diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 0161c20..211c825 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -222,8 +222,8 @@ public static final String HIVE_ADDED_JARS = "hive.added.jars"; public static final String VECTOR_MODE = "VECTOR_MODE"; public static final String USE_VECTORIZED_INPUT_FILE_FORMAT = "USE_VECTORIZED_INPUT_FILE_FORMAT"; - public static String MAPNAME = "Map "; - public static String REDUCENAME = "Reducer "; + public static String MAPNAME = "Vertex "; + public static String REDUCENAME = "Vertex "; /** * ReduceField: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index aa2dfc7..eadcf93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -273,7 +273,7 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, case BROADCAST_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); @@ -288,11 +288,11 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, break; } - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: setupAutoReducerParallelism(edgeProp, w); // fall through @@ -319,7 +319,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr throws IOException { switch(edgeProp.getEdgeType()) { - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); @@ -332,7 +332,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr w.setVertexManagerPlugin(desc); break; } - case SIMPLE_EDGE: { + case SORT_PARTITION_EDGE: { setupAutoReducerParallelism(edgeProp, w); break; } @@ -364,7 +364,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et1Conf.createDefaultBroadcastEdgeProperty(); - case CUSTOM_EDGE: + case CO_PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig @@ -382,7 +382,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig @@ -392,7 +392,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et3Conf.createDefaultEdgeProperty(); - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: default: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index 3a6baca..54bd63c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -252,10 +252,10 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma ReduceSinkOperator.class); if (rootOp == null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } else { Operator rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted( @@ -263,16 +263,16 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TableScanOperator.class); if (rootOp != null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } } else if (mapJoinOp.getConf().isDynamicPartitionHashJoin()) { - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { // disable auto parallelism for bucket map joins parentRS.getConf().setReducerTraits(EnumSet.of(FIXED)); } @@ -284,7 +284,7 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TezWork tezWork = context.currentTask.getWork(); LOG.debug("connecting "+parentWork.getName()+" with "+myWork.getName()); tezWork.connect(parentWork, myWork, edgeProp); - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(myWork, VertexType.INITIALIZED_EDGES); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index e2363eb..ff5bf82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -501,10 +501,10 @@ public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork follo if (reduceWork.getReducer() instanceof MapJoinOperator) { MapJoinOperator joinOp = (MapJoinOperator) reduceWork.getReducer(); if (joinOp.getConf().isDynamicPartitionHashJoin()) { - return EdgeType.CUSTOM_SIMPLE_EDGE; + return EdgeType.PARTITION_EDGE; } } } - return EdgeType.SIMPLE_EDGE; + return EdgeType.SORT_PARTITION_EDGE; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 2b96e51..b604930 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -283,7 +283,7 @@ public Object process(Node nd, Stack stack, LOG.debug("connecting "+parentWork.getName()+" with "+work.getName()); TezEdgeProperty edgeProp = parentWorkMap.getValue(); tezWork.connect(parentWork, work, edgeProp); - if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) { + if (edgeProp.getEdgeType() == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(work, VertexType.INITIALIZED_EDGES); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index a3aa12f..21ae21e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -23,11 +23,11 @@ public class TezEdgeProperty { public enum EdgeType { - SIMPLE_EDGE, + SORT_PARTITION_EDGE, BROADCAST_EDGE, CONTAINS, - CUSTOM_EDGE, - CUSTOM_SIMPLE_EDGE, + CO_PARTITION_EDGE, + PARTITION_EDGE, } private HiveConf hiveConf; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 5c012f3..c76c077 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -153,7 +153,7 @@ public Edge answer(InvocationOnMock invocation) throws Throwable { rws[0].setReducer(op); rws[1].setReducer(op); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(mws[0], rws[0], edgeProp); work.connect(mws[1], rws[0], edgeProp); work.connect(rws[0], rws[1], edgeProp); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java index c1e192c..ea72cb1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java @@ -67,7 +67,7 @@ public void testConnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork child = nodes.get(1); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, child, edgeProp); Assert.assertEquals(work.getParents(child).size(), 1); @@ -84,7 +84,7 @@ public void testConnect() throws Exception { Assert.assertEquals(work.getChildren(w).size(), 0); } - Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SIMPLE_EDGE); + Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SORT_PARTITION_EDGE); } @Test @@ -117,7 +117,7 @@ public void testDisconnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -136,7 +136,7 @@ public void testRemove() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -151,7 +151,7 @@ public void testRemove() throws Exception { @Test public void testGetAllWork() throws Exception { - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); for (int i = 4; i > 0; --i) { work.connect(nodes.get(i), nodes.get(i-1), edgeProp); } diff --git a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index 3649e56..2838297 100644 --- a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acidtbldefault diff --git a/ql/src/test/results/clientpositive/llap/auto_join0.q.out b/ql/src/test/results/clientpositive/llap/auto_join0.q.out index 3dda24a..6140878 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join0.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join0.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: -- SORT_QUERY_RESULTS explain @@ -34,13 +34,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -58,25 +58,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -91,13 +73,13 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Reducer 6 + 1 Vertex 6 Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -113,7 +95,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -128,7 +110,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -146,7 +146,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( SELECT src1.key as k1, src1.value as v1, diff --git a/ql/src/test/results/clientpositive/llap/auto_join1.q.out b/ql/src/test/results/clientpositive/llap/auto_join1.q.out index 3624062..fb05cf3 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join1.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join1.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -51,7 +51,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -71,7 +71,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) diff --git a/ql/src/test/results/clientpositive/llap/auto_join21.q.out b/ql/src/test/results/clientpositive/llap/auto_join21.q.out index 91ea004..e09512f 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join21.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join21.q.out @@ -17,11 +17,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -34,36 +34,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -88,7 +59,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -102,6 +73,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_join29.q.out b/ql/src/test/results/clientpositive/llap/auto_join29.q.out index 94134c5..78a0bee 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join29.q.out @@ -17,11 +17,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -34,36 +34,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -88,7 +59,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -102,6 +73,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -632,11 +632,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -649,39 +649,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -706,7 +674,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 322 Data size: 171948 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -720,6 +688,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1250,11 +1250,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1270,36 +1270,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1324,7 +1295,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 974 Data size: 520116 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1338,6 +1309,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1880,11 +1880,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1900,33 +1900,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1951,7 +1925,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1965,6 +1939,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2507,11 +2507,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2527,39 +2527,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2580,7 +2548,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2594,6 +2562,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2624,11 +2624,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2644,36 +2644,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2698,7 +2669,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2712,6 +2683,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3242,11 +3242,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3265,8 +3265,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3278,7 +3278,21 @@ STAGE PLANS: Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3294,7 +3308,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3310,20 +3324,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3354,11 +3354,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3374,7 +3374,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -3396,8 +3396,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3409,7 +3409,21 @@ STAGE PLANS: Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3425,20 +3439,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3491,11 +3491,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3513,8 +3513,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3526,7 +3526,21 @@ STAGE PLANS: Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3542,7 +3556,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3558,20 +3572,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_join30.q.out b/ql/src/test/results/clientpositive/llap/auto_join30.q.out index d2cfa62..8d7d414 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join30.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join30.q.out @@ -23,12 +23,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -46,7 +46,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -65,19 +77,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -92,7 +92,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -103,7 +103,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -169,12 +169,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -189,23 +189,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -220,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -231,7 +215,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -246,7 +230,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -310,12 +310,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -330,7 +330,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -346,19 +358,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -373,7 +373,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -384,7 +384,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,13 +456,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -480,26 +480,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -518,19 +511,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -547,8 +528,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 - 2 Reducer 7 + 0 Vertex 2 + 2 Vertex 7 Statistics: Num rows: 2974 Data size: 529372 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -559,7 +540,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -574,7 +555,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,14 +649,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -671,39 +671,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -715,7 +683,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -737,7 +705,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -752,7 +720,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -765,7 +749,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -840,14 +840,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -862,39 +862,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -906,7 +874,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -928,7 +896,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -943,7 +911,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -956,7 +940,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1031,14 +1031,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1053,39 +1053,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1097,7 +1065,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1119,7 +1087,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1134,7 +1102,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1147,7 +1131,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1222,14 +1222,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1244,39 +1244,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1288,7 +1256,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1310,7 +1278,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1325,7 +1293,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1338,7 +1322,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_join_filters.q.out b/ql/src/test/results/clientpositive/llap/auto_join_filters.q.out index d1d9408..f4c1e79 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join_filters.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -44,7 +44,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -310,7 +310,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -320,7 +320,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -330,7 +330,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out b/ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out index 5984e8f..75254df 100644 --- a/ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index 3083291..8f51df2 100644 --- a/ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -63,10 +63,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -106,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -174,11 +174,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -220,7 +220,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -239,7 +239,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -333,12 +333,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -382,7 +382,44 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: b @@ -426,44 +463,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -556,10 +556,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -599,7 +599,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -677,10 +677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -720,7 +720,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -822,10 +822,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -865,7 +865,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -957,10 +957,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1000,7 +1000,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1068,11 +1068,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1091,26 +1091,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1129,7 +1110,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1144,6 +1125,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1193,10 +1193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1236,7 +1236,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1306,10 +1306,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1362,7 +1362,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1448,10 +1448,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1491,7 +1491,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1591,7 +1591,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1809,10 +1809,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1868,7 +1868,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out index 9a586fa..2c1f92d 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out @@ -118,11 +118,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -198,7 +198,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -215,12 +215,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -337,7 +337,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -405,11 +405,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -426,12 +426,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -548,7 +548,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -624,37 +655,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -692,11 +692,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -713,12 +713,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -835,7 +835,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -911,37 +942,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out index 4a049c7..a36dbd6 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out @@ -73,12 +73,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -97,7 +99,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -110,7 +112,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -129,7 +146,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -142,7 +159,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -166,23 +183,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -244,11 +244,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -268,7 +268,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -287,7 +287,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -300,7 +300,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out index 5cb3db5..056f342 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out @@ -114,11 +114,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -193,7 +193,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -210,12 +210,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -330,7 +330,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -406,11 +406,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -485,7 +485,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -502,12 +502,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -622,7 +622,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -692,11 +692,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -767,7 +767,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -780,12 +780,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 key (type: string) 1 key (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -900,7 +900,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -970,11 +970,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1045,7 +1045,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1059,14 +1059,14 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 0 to 2 - Estimated key counts: Map 1 => 1, Map 4 => 116 + Estimated key counts: Vertex 1 => 1, Vertex 4 => 116 keys: 0 key (type: string) 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Position of Big Table: 1 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1181,7 +1181,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Map 4 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1299,37 +1330,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 1bef238..ad791d3 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -138,7 +138,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -152,11 +152,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -232,7 +232,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -308,7 +308,7 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [b] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -326,25 +326,25 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 1 to 2 - Estimated key counts: Map 1 => 1, Map 2 => 1 + Estimated key counts: Vertex 1 => 1, Vertex 2 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) input vertices: - 0 Map 1 - 1 Map 2 + 0 Vertex 1 + 1 Vertex 2 Position of Big Table: 2 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 5 => 1 + Estimated key counts: Vertex 5 => 1 keys: 0 1 input vertices: - 1 Map 5 + 1 Vertex 5 Position of Big Table: 0 Statistics: Num rows: 255 Data size: 29909 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -461,7 +461,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: d @@ -529,37 +560,6 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [d] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -567,7 +567,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out index 633abff..5ebb8eb 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -108,7 +108,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -136,7 +136,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -303,10 +303,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -326,7 +326,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -354,7 +354,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -521,10 +521,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -544,7 +544,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -572,7 +572,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out index 9bab958..6eb8b31 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out @@ -51,11 +51,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -71,7 +71,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -84,7 +84,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -100,21 +115,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -160,11 +160,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -180,7 +180,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -196,7 +196,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 207 Data size: 2080 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -209,7 +209,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out index 8059b1c..cb0a6d2 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out @@ -49,11 +49,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -69,7 +69,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -82,7 +82,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -98,21 +113,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -135,11 +135,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -155,7 +155,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -171,7 +171,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -184,7 +184,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out index 0b1d648..e9f33c3 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out @@ -98,11 +98,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -119,12 +119,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -241,7 +241,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -317,37 +348,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,12 +408,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -530,7 +530,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -606,37 +637,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out index d99d425..18d0326 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out @@ -98,11 +98,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -226,7 +226,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -243,12 +243,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -317,7 +317,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -385,11 +385,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -406,12 +406,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -480,7 +480,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -604,37 +635,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -672,11 +672,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -693,12 +693,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -767,7 +767,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -891,37 +922,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out index 069f08d..25cef42 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out @@ -114,11 +114,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -242,7 +242,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -259,12 +259,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -333,7 +333,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -401,11 +401,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -422,12 +422,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -496,7 +496,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -620,37 +651,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -688,11 +688,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -709,12 +709,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -783,7 +783,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -907,37 +938,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out index 5e45f9c..02c8e70 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -236,7 +236,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -298,10 +298,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -449,7 +449,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -511,11 +511,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -532,12 +532,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -605,7 +605,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -680,37 +711,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out index 9e83db1..3bf4490 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out @@ -97,11 +97,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -138,26 +138,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -176,7 +157,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -191,6 +172,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -230,11 +230,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -271,26 +271,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: d - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -309,7 +290,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -324,6 +305,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -363,11 +363,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -404,26 +404,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: h - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -442,7 +423,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,6 +438,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: h + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -492,11 +492,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -533,26 +533,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -571,7 +552,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -586,6 +567,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -621,10 +621,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -677,7 +677,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -727,11 +727,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -768,26 +768,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -806,7 +787,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -821,6 +802,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -856,11 +856,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -897,26 +897,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -935,7 +916,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -950,6 +931,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -985,11 +985,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1026,26 +1026,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1064,7 +1045,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1079,6 +1060,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1114,10 +1114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1170,7 +1170,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1220,11 +1220,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1261,26 +1261,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1299,7 +1280,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1314,6 +1295,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out index 3e4f408..1d7c779 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out @@ -131,11 +131,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -398,7 +398,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -468,11 +468,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -489,12 +489,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -611,7 +611,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -735,37 +766,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -805,11 +805,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -826,12 +826,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -948,7 +948,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1072,37 +1103,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out index dae32a5..b5758f9 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out @@ -131,11 +131,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -398,7 +398,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -468,11 +468,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -489,12 +489,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -611,7 +611,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -735,37 +766,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -807,11 +807,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -828,12 +828,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -950,7 +950,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1074,37 +1105,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out index 9549778..0fec632 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out @@ -63,11 +63,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -86,7 +86,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -99,7 +99,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -118,21 +133,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -180,11 +180,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -204,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -220,7 +220,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -239,22 +255,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -317,12 +317,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -342,7 +342,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -356,26 +356,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -394,7 +375,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -409,6 +390,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -488,13 +488,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -514,7 +514,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -530,7 +530,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -549,7 +579,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -569,7 +599,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -585,7 +615,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: b @@ -604,51 +649,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -727,11 +727,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -750,7 +750,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -763,7 +763,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -782,21 +797,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -860,11 +860,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -883,7 +883,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -896,7 +896,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -915,21 +930,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1017,11 +1017,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1040,7 +1040,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1053,7 +1053,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1072,21 +1087,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1164,11 +1164,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1187,7 +1187,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1200,7 +1200,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1219,21 +1234,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1287,11 +1287,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1310,7 +1310,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1323,7 +1323,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1342,21 +1357,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1406,11 +1406,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1429,7 +1429,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1442,7 +1442,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1461,21 +1476,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1523,11 +1523,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1546,7 +1546,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1559,7 +1559,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1578,21 +1593,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1650,11 +1650,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1675,8 +1675,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1689,7 +1689,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1708,7 +1723,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1727,21 +1742,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1813,11 +1813,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1836,7 +1836,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1849,7 +1849,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1868,21 +1883,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1942,11 +1942,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1965,7 +1965,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1978,7 +1978,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1997,21 +2012,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2059,11 +2059,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2083,7 +2083,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2099,7 +2099,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2118,22 +2134,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2196,12 +2196,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2221,7 +2221,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -2235,26 +2235,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2273,7 +2254,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2288,6 +2269,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2367,13 +2367,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2393,7 +2393,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2409,7 +2409,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2428,7 +2458,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -2448,72 +2478,23 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2528,6 +2509,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2606,11 +2606,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2629,7 +2629,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2642,7 +2642,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2661,21 +2676,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2739,11 +2739,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2762,7 +2762,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2775,7 +2775,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2794,21 +2809,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2896,11 +2896,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2919,7 +2919,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2932,7 +2932,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2951,21 +2966,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3043,11 +3043,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3066,7 +3066,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3079,7 +3079,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3098,21 +3113,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3162,11 +3162,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3185,7 +3185,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3198,7 +3198,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3217,21 +3232,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3279,11 +3279,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3302,7 +3302,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3315,7 +3315,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3334,21 +3349,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3406,11 +3406,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3431,8 +3431,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3445,7 +3445,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3464,7 +3479,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -3483,21 +3498,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3569,11 +3569,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3592,7 +3592,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3605,7 +3605,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3624,21 +3639,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket2.q.out b/ql/src/test/results/clientpositive/llap/bucket2.q.out index 244f247..615e0df 100644 --- a/ql/src/test/results/clientpositive/llap/bucket2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -101,7 +101,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/bucket3.q.out b/ql/src/test/results/clientpositive/llap/bucket3.q.out index dcf731d..e54dde4 100644 --- a/ql/src/test/results/clientpositive/llap/bucket3.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -101,7 +101,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/bucket4.q.out b/ql/src/test/results/clientpositive/llap/bucket4.q.out index 4504177..2b31aa5 100644 --- a/ql/src/test/results/clientpositive/llap/bucket4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -98,7 +98,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/bucket5.q.out b/ql/src/test/results/clientpositive/llap/bucket5.q.out index 708d1a0..25fd0a3 100644 --- a/ql/src/test/results/clientpositive/llap/bucket5.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -48,11 +48,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -135,7 +135,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -176,7 +176,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/bucket6.q.out b/ql/src/test/results/clientpositive/llap/bucket6.q.out index 20895f8..d25a965 100644 --- a/ql/src/test/results/clientpositive/llap/bucket6.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket6.q.out @@ -23,10 +23,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -43,7 +43,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out index 7725869..260a35b 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out @@ -55,11 +55,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -83,7 +83,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -98,7 +98,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -185,11 +185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -213,7 +213,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -228,7 +228,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -289,10 +289,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -316,7 +316,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -370,10 +370,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -397,7 +397,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -453,11 +453,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -481,7 +481,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -496,7 +496,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -557,11 +557,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -585,7 +585,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -600,7 +600,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -659,10 +659,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -685,7 +685,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1044,10 +1044,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1070,7 +1070,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1162,11 +1162,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1191,7 +1191,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1206,7 +1206,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1265,11 +1265,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1293,7 +1293,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1308,7 +1308,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1470,11 +1470,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1499,7 +1499,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1514,7 +1514,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1573,11 +1573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1601,7 +1601,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1620,7 +1620,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket_many.q.out b/ql/src/test/results/clientpositive/llap/bucket_many.q.out index 8433022..6b531f2 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -97,7 +97,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 20702f9..124b430 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -125,10 +125,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,7 +148,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -168,7 +168,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -209,11 +209,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -233,26 +233,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -271,7 +252,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -282,7 +263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -297,6 +278,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -346,11 +346,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 4 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -389,7 +389,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -398,7 +398,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -411,7 +411,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d @@ -430,21 +445,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -498,11 +498,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 2 (CO_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d @@ -521,7 +521,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -540,7 +540,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -560,7 +560,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -569,7 +569,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -582,7 +582,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -650,11 +650,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -680,7 +680,26 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -700,7 +719,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) @@ -715,25 +734,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -784,7 +784,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), substr(_col2, 5) (type: string) @@ -804,7 +804,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: tab @@ -824,7 +858,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -844,40 +878,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -904,11 +904,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -928,7 +928,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), substr(_col1, 5) (type: string) @@ -948,7 +948,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -967,7 +1001,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -987,40 +1021,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1047,10 +1047,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1072,8 +1072,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 1100 Data size: 20486 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1088,7 +1088,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1108,7 +1108,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1151,10 +1151,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1174,7 +1174,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1184,7 +1184,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1199,7 +1199,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -1218,7 +1218,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1266,11 +1266,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1296,7 +1296,22 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1316,7 +1331,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1331,21 +1346,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1372,11 +1372,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1396,7 +1396,22 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1416,7 +1431,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1431,21 +1446,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1472,10 +1472,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1495,7 +1495,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1515,7 +1515,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1572,10 +1572,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1595,7 +1595,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1615,7 +1615,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1650,10 +1650,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1673,7 +1673,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1683,7 +1683,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -1698,7 +1698,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -1717,7 +1717,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1761,10 +1761,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1796,10 +1796,10 @@ STAGE PLANS: Target Input: b Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Target Vertex: Map 2 + Target Vertex: Vertex 2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1819,7 +1819,7 @@ STAGE PLANS: 1 _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60500 Data size: 12826000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string) @@ -1872,11 +1872,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1895,14 +1895,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1921,21 +1936,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1991,11 +1991,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2014,14 +2014,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2040,21 +2055,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out index f6500eb..ded7ccd 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out @@ -121,10 +121,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -144,7 +144,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -154,7 +154,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -169,7 +169,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -188,7 +188,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -254,10 +254,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -277,7 +277,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -334,10 +334,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -357,7 +357,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 60 Data size: 1126 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -368,7 +368,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -409,10 +409,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -432,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -443,7 +443,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 1024 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -526,7 +526,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -557,10 +557,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -580,26 +580,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -615,7 +596,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -624,6 +605,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -644,10 +644,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -667,27 +667,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -703,7 +683,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string) @@ -716,6 +696,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out b/ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out index 163e819..04d47fd 100644 --- a/ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out @@ -22,7 +22,7 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@T2 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM ( SELECT tmp1.name as name FROM ( SELECT name, 'MMM' AS n FROM T1) tmp1 diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index e9bb701..e52e5c7 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -67,28 +67,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -127,6 +106,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -167,10 +167,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -191,28 +191,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -251,6 +230,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -385,10 +385,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,7 +458,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -530,54 +578,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -737,10 +737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -810,7 +810,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -882,54 +930,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index 84de3e3..1423edc 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -111,10 +111,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -186,7 +186,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -258,54 +306,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -469,10 +469,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -544,7 +544,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -616,54 +664,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -846,10 +846,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -921,7 +921,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1040,54 +1088,6 @@ STAGE PLANS: Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] /srcbucket_mapjoin_part_2/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index 475a9e3..65f5182 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -210,7 +210,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -282,54 +330,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -493,10 +493,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -568,7 +568,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -640,54 +688,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index 4815a1c..a217ab0 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -208,7 +208,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -278,54 +326,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -550,7 +550,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -620,54 +668,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out index b515af6..4d84682 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out @@ -77,11 +77,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -153,7 +153,66 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_1 Truncated Path -> Alias: /srcbucket_mapjoin_part_1/ds=2008-04-08/hr=0 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col8 + Position of Big Table: 0 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: aa + sort order: ++ + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 1 + TopN Hash Memory Usage: 0.1 + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -226,65 +285,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08/hr=0 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col8 - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: aa - sort order: ++ - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 1 - TopN Hash Memory Usage: 0.1 - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out index f6ddfe9..2278ea6 100644 --- a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -97,7 +97,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -159,7 +159,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -221,7 +221,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -283,7 +283,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -345,7 +345,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -407,7 +407,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -469,7 +469,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -531,7 +531,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -593,7 +593,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -655,7 +655,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -717,7 +717,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -779,7 +779,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -840,7 +840,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -905,7 +905,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -971,7 +971,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1032,7 +1032,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1093,7 +1093,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1154,7 +1154,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1219,7 +1219,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1280,7 +1280,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1341,7 +1341,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1406,7 +1406,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1466,7 +1466,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1527,7 +1527,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1588,7 +1588,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned @@ -1649,7 +1649,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned diff --git a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index 8519ff3..0c008fa 100644 --- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -105,11 +105,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -129,7 +129,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -149,7 +149,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -163,7 +163,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -288,11 +288,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -312,7 +312,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -332,7 +332,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -346,7 +346,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -495,11 +495,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -519,7 +519,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -539,7 +539,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 184 Data size: 3097 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -553,7 +553,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -706,11 +706,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -730,7 +730,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -899,11 +899,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -923,7 +923,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -943,7 +943,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -957,7 +957,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1092,11 +1092,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -1116,7 +1116,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -1136,7 +1136,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) @@ -1150,7 +1150,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out index 4b4a95e..0030f72 100644 --- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out @@ -83,11 +83,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -107,7 +107,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -127,7 +127,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -273,11 +273,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -297,7 +297,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -317,7 +317,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -331,7 +331,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -463,11 +463,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -487,7 +487,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -507,7 +507,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -521,7 +521,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -588,11 +588,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -612,7 +612,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -632,7 +632,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -646,7 +646,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -719,11 +719,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -743,7 +743,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -763,7 +763,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -777,7 +777,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -927,11 +927,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -951,7 +951,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -971,7 +971,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -985,7 +985,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1145,11 +1145,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1169,7 +1169,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1189,7 +1189,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -1203,7 +1203,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out index 11de932..b3a9a26 100644 --- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out @@ -83,11 +83,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -107,7 +107,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -127,7 +127,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -268,11 +268,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -292,7 +292,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -312,7 +312,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -326,7 +326,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -459,11 +459,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -483,7 +483,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -503,7 +503,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 30 Data size: 515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -517,7 +517,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/cluster.q.out b/ql/src/test/results/clientpositive/llap/cluster.q.out index 8c89ee3..d0ee03a 100644 --- a/ql/src/test/results/clientpositive/llap/cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/cluster.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -36,7 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -149,10 +149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -172,7 +172,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -217,10 +217,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -240,7 +240,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -285,10 +285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -308,7 +308,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -353,10 +353,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -376,7 +376,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,10 +421,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -444,7 +444,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -489,10 +489,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -512,7 +512,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -557,11 +557,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -577,22 +577,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -613,7 +598,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -627,6 +612,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -658,11 +658,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -678,23 +678,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -715,7 +699,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -729,6 +713,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -780,23 +780,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -817,7 +801,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -831,6 +815,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -862,11 +862,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -882,22 +882,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -918,7 +903,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -932,6 +917,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -975,12 +975,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1000,7 +1002,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -1020,22 +1036,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index 3859496..a649891 100644 --- a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -331,7 +331,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -508,10 +508,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -531,7 +531,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -717,7 +717,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -870,10 +870,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -893,7 +893,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/column_access_stats.q.out b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out index 5a66b0d..af13f3b 100644 --- a/ql/src/test/results/clientpositive/llap/column_access_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out @@ -183,11 +183,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -205,7 +207,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -223,8 +225,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -271,11 +271,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -293,7 +295,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -311,8 +313,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -388,10 +388,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -410,7 +410,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -429,24 +447,6 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -521,10 +521,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -544,7 +544,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -564,24 +582,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -623,10 +623,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -645,7 +645,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -664,24 +682,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -737,11 +737,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -760,7 +760,41 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: t2 @@ -779,7 +813,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: t3 @@ -799,40 +833,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out index ab0b36c..c20992a 100644 --- a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out +++ b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out @@ -41,47 +41,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Union 4 (CONTAINS) -Map 6 <- Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 5 <- Union 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 4 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_17] - Merge Join Operator [MERGEJOIN_21] (rows=1 width=0) - Conds:RS_13._col0=RS_14._col0(Left Outer),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_0] (rows=1 width=0) - default@tb1,a,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Limit [LIM_11] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - <-Union 4 [SIMPLE_EDGE] - <-Map 3 [CONTAINS] llap - Reduce Output Operator [RS_9] - Limit [LIM_8] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_3] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_2] (rows=1 width=0) - Output:["id"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_9] - Limit [LIM_8] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_5] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_4] (rows=1 width=0) + Vertex 2 diff --git a/ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out index 98d2328..763f217 100644 --- a/ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out +++ b/ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out @@ -54,38 +54,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=5 width=22) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8._col3=RS_9._col0(Left Semi),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2","_col3"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((val = 't1val01') and dimid is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_16] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01' PREHOOK: type: QUERY @@ -107,53 +82,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=5 width=24) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) - Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2"] - Filter Operator [FIL_24] (rows=5 width=20) - predicate:((val = 't1val01') and id is not null and dimid is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_25] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 3 PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id PREHOOK: type: QUERY @@ -177,53 +113,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=5 width=24) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) - Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2"] - Filter Operator [FIL_24] (rows=5 width=20) - predicate:((val = 't1val01') and dimid is not null and id is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_25] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 3 PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id PREHOOK: type: QUERY @@ -247,36 +144,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=1 width=20) - predicate:false - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100 PREHOOK: type: QUERY @@ -295,36 +169,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200) PREHOOK: type: QUERY @@ -345,36 +196,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=1 width=20) - predicate:false - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200 PREHOOK: type: QUERY @@ -393,36 +221,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100 PREHOOK: type: QUERY @@ -443,36 +248,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out index 5702c6e..94b7955 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out @@ -29,12 +29,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -53,26 +53,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -95,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -117,7 +98,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -132,6 +113,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -177,12 +177,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -201,26 +201,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -243,7 +224,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -265,7 +246,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -280,6 +261,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -333,12 +333,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -357,7 +357,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -377,7 +377,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -393,7 +393,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -415,7 +415,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -481,12 +481,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -505,31 +505,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -552,7 +528,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -574,7 +550,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -589,6 +565,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -634,12 +634,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -658,31 +658,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -705,7 +681,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -727,7 +703,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -742,6 +718,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -793,12 +793,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -814,23 +814,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -853,7 +837,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -875,7 +859,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -890,6 +874,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -935,12 +935,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -956,23 +956,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -995,7 +979,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1017,7 +1001,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1032,6 +1016,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1083,12 +1083,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1104,23 +1104,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1147,7 +1131,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1169,7 +1153,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1184,6 +1168,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1229,12 +1229,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1250,23 +1250,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1293,7 +1277,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1315,7 +1299,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1330,6 +1314,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1379,11 +1379,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1399,23 +1399,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1442,7 +1426,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1458,6 +1442,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1514,11 +1514,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1534,23 +1534,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1577,7 +1561,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1593,6 +1577,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1657,12 +1657,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1678,23 +1678,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1721,7 +1705,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1743,7 +1727,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1758,6 +1742,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1803,12 +1803,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1824,23 +1824,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1867,7 +1851,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1889,7 +1873,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1904,6 +1888,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1955,12 +1955,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1976,23 +1976,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2015,7 +1999,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2037,7 +2021,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2052,6 +2036,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2097,12 +2097,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2118,23 +2118,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2157,7 +2141,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2179,7 +2163,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2194,6 +2178,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2247,12 +2247,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2268,23 +2268,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2307,7 +2291,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2329,7 +2313,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2344,6 +2328,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2389,12 +2389,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2410,23 +2410,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2449,7 +2433,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2471,7 +2455,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2486,6 +2470,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2533,12 +2533,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2558,26 +2558,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2600,7 +2581,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2622,7 +2603,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2637,6 +2618,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2682,12 +2682,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2707,26 +2707,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2749,7 +2730,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2771,7 +2752,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2786,6 +2767,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2833,12 +2833,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2857,26 +2857,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2899,7 +2880,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2921,7 +2902,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2936,6 +2917,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2981,12 +2981,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3005,26 +3005,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3047,7 +3028,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3069,7 +3050,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3084,6 +3065,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index dfb4804..c0a5ebd 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -33,13 +33,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -61,29 +61,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -98,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -122,7 +100,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -137,7 +115,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -201,13 +201,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -229,29 +229,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -266,7 +244,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -290,7 +268,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -305,7 +283,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -371,13 +371,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -400,26 +400,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -434,7 +415,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -458,7 +439,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -473,7 +454,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -537,13 +537,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -566,26 +566,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -600,7 +581,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -624,7 +605,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -639,7 +620,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -705,13 +705,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -730,30 +730,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -768,7 +745,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -792,7 +769,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -807,7 +784,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -871,13 +871,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -896,30 +896,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -934,7 +911,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -958,7 +935,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -973,7 +950,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1039,13 +1039,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1068,30 +1068,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1106,7 +1083,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1130,7 +1107,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1145,7 +1122,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1209,13 +1209,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1238,30 +1238,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1276,7 +1253,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1300,7 +1277,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1315,7 +1292,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1381,14 +1381,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1409,28 +1409,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1443,7 +1422,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1466,7 +1445,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1488,7 +1467,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1503,7 +1482,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1575,14 +1575,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1603,28 +1603,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1637,7 +1616,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1660,7 +1639,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1682,7 +1661,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1697,7 +1676,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1769,13 +1769,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1794,49 +1794,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1857,7 +1815,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1881,7 +1839,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1896,7 +1854,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1960,13 +1960,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1985,49 +1985,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2048,7 +2006,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2072,7 +2030,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2087,7 +2045,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index 74b3d6c..dbd8e4b 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -35,14 +35,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -61,65 +61,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -140,7 +82,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -164,7 +106,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -179,7 +121,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -206,7 +187,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -221,6 +202,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -270,53 +270,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -335,26 +296,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -375,7 +317,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -399,7 +341,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -414,7 +356,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -441,7 +422,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,6 +437,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -507,13 +507,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -533,7 +533,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -547,7 +547,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -564,7 +564,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -584,7 +599,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -604,7 +619,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -624,7 +639,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -643,36 +673,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -707,68 +707,29 @@ FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value ON b.key = d.key) tmp PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) -FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value - FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b - JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d - ON b.key = d.key) tmp -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 +SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) +FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value + FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b + JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d + ON b.key = d.key) tmp +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: y @@ -787,26 +748,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -827,7 +769,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -851,7 +793,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -866,7 +808,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -893,7 +874,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -908,6 +889,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -957,53 +957,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1022,26 +983,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1062,7 +1004,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1086,7 +1028,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1101,7 +1043,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1128,7 +1109,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1143,6 +1124,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1194,13 +1194,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1220,7 +1220,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -1234,7 +1234,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col2) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -1251,7 +1251,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -1271,7 +1286,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -1291,7 +1306,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -1311,7 +1326,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -1330,36 +1360,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out index 67e636b..cb63eb5 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out @@ -75,12 +75,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -99,45 +99,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -166,7 +128,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -188,7 +150,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -203,6 +165,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -250,12 +250,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -274,45 +274,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -341,7 +303,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -363,7 +325,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -378,6 +340,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -427,12 +427,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -454,8 +454,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 4 - 2 Map 5 + 1 Vertex 4 + 2 Vertex 5 Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -475,7 +475,44 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hash(_col0) (type: int), hash(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col0), sum(_col1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y @@ -494,7 +531,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z @@ -513,43 +550,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hash(_col0) (type: int), hash(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(_col0), sum(_col1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -601,12 +601,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -622,39 +622,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -679,7 +647,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -701,7 +669,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -716,6 +684,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -763,12 +763,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -784,39 +784,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -841,7 +809,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -863,7 +831,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -878,6 +846,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -931,12 +931,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -952,39 +952,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1013,7 +981,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1035,7 +1003,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1050,6 +1018,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1101,12 +1101,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1122,39 +1122,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1183,7 +1151,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1205,21 +1173,53 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1267,12 +1267,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1288,39 +1288,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1349,7 +1317,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1371,7 +1339,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1386,6 +1354,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1439,12 +1439,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1460,39 +1460,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1521,7 +1489,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1543,7 +1511,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1558,6 +1526,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1609,12 +1609,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1630,39 +1630,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1691,7 +1659,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1713,7 +1681,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1728,6 +1696,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1775,12 +1775,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1796,39 +1796,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1857,7 +1825,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1879,7 +1847,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1894,6 +1862,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index cf66d25..d343e1c 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -33,14 +33,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -59,64 +59,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -139,7 +82,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -154,7 +97,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -172,7 +115,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -195,7 +176,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -210,6 +191,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -277,14 +277,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -303,64 +303,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -383,7 +326,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -398,7 +341,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -416,7 +359,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -439,7 +420,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -454,6 +435,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -523,13 +523,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -549,7 +549,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -565,7 +565,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -584,7 +599,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -604,7 +619,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -620,7 +635,33 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 0 Vertex 2 + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -639,47 +680,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -751,11 +751,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -774,7 +774,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -800,25 +818,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -896,11 +896,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -919,7 +919,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -945,25 +963,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1047,12 +1047,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1071,29 +1071,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1109,25 +1108,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1150,7 +1131,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1165,6 +1146,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1228,12 +1228,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1252,29 +1252,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1290,25 +1289,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1331,7 +1312,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1346,6 +1327,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1415,12 +1415,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1439,45 +1439,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1500,7 +1462,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1515,7 +1477,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1533,6 +1495,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1618,12 +1618,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1642,45 +1642,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1703,7 +1665,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1718,7 +1680,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1736,6 +1698,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1829,12 +1829,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -1853,7 +1853,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -1872,7 +1896,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1891,50 +1915,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1957,7 +1938,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1972,6 +1953,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2067,12 +2067,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -2091,7 +2091,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -2110,7 +2134,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2129,50 +2153,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2195,7 +2176,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2210,6 +2191,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2305,12 +2305,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2329,64 +2329,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: zz - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2409,7 +2352,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2424,7 +2367,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2448,6 +2391,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2509,81 +2509,43 @@ POSTHOOK: Input: default@src1 406 406 16 406 406 16 406 406 16 -66 66 1 -98 98 4 -98 98 4 -PREHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 +66 66 1 +98 98 4 +98 98 4 +PREHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: zz + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -2599,26 +2561,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2641,7 +2584,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2656,7 +2599,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2680,6 +2623,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2777,14 +2777,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2806,49 +2806,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2863,7 +2821,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2886,7 +2844,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2901,7 +2859,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2923,7 +2881,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2938,6 +2918,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3031,14 +3031,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3060,49 +3060,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3117,7 +3075,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3140,7 +3098,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3155,7 +3113,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3177,7 +3135,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3192,6 +3172,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3291,15 +3291,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3318,7 +3318,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -3337,46 +3337,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3399,7 +3360,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3414,7 +3375,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3429,7 +3390,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) sort order: +++++ Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3443,7 +3404,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3466,7 +3466,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3548,14 +3548,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3574,65 +3574,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3655,7 +3597,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3670,7 +3612,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3688,7 +3630,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3711,7 +3692,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3726,6 +3707,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3793,13 +3793,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3819,7 +3819,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3835,7 +3835,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -3854,7 +3869,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -3874,7 +3889,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3890,41 +3905,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3941,7 +3922,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -3950,6 +3931,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/count.q.out b/ql/src/test/results/clientpositive/llap/count.q.out index 5c38049..04cabf6 100644 --- a/ql/src/test/results/clientpositive/llap/count.q.out +++ b/ql/src/test/results/clientpositive/llap/count.q.out @@ -44,10 +44,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -70,7 +70,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -118,10 +118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -143,7 +143,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -187,10 +187,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -255,10 +255,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -273,7 +273,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -319,10 +319,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -344,7 +344,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -388,11 +388,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -408,7 +408,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -422,7 +422,7 @@ STAGE PLANS: Map-reduce partition columns: a (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -474,11 +474,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -494,7 +494,7 @@ STAGE PLANS: Statistics: Num rows: 19 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -507,7 +507,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: b (type: int) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -559,11 +559,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -579,7 +579,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -593,7 +593,7 @@ STAGE PLANS: Map-reduce partition columns: c (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -648,10 +648,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -668,7 +668,7 @@ STAGE PLANS: value expressions: b (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -725,10 +725,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -745,7 +745,7 @@ STAGE PLANS: value expressions: $f3 (type: int), $f4 (type: int), $f5 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -815,7 +815,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -865,10 +865,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -883,7 +883,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/cross_join.q.out b/ql/src/test/results/clientpositive/llap/cross_join.q.out index 8578dbf..d62716d 100644 --- a/ql/src/test/results/clientpositive/llap/cross_join.q.out +++ b/ql/src/test/results/clientpositive/llap/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- current explain select src.key from src join src src2 PREHOOK: type: QUERY @@ -14,10 +14,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -32,19 +32,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -62,6 +50,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -69,7 +69,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- ansi cross join explain select src.key from src cross join src src2 PREHOOK: type: QUERY @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -103,19 +103,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -133,6 +121,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -155,10 +155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -177,7 +177,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -196,24 +214,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -221,7 +221,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -235,10 +235,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -255,7 +255,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -266,7 +266,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -285,7 +285,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 @@ -299,10 +299,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -319,7 +319,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -330,7 +330,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -385,7 +385,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -396,7 +396,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 diff --git a/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out b/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out index be303c4..0635f72 100644 --- a/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out +++ b/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -64,22 +64,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -97,6 +82,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -104,7 +104,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -118,11 +118,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -142,7 +142,40 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -162,7 +195,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -177,39 +210,6 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -217,7 +217,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -237,12 +237,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -257,29 +257,28 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: d2 + alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -295,25 +294,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -334,7 +315,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -346,6 +327,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -353,8 +353,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -368,12 +368,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -388,34 +388,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -433,7 +406,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -454,7 +442,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 100 Data size: 1540 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -466,6 +454,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 770 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -473,7 +473,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -491,13 +491,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -518,45 +518,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -568,7 +530,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -586,7 +548,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -607,7 +588,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -619,6 +600,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out index 3b14b84..58192cb 100644 --- a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -66,7 +66,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 5000 Data size: 106120 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -77,7 +77,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -99,7 +99,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -113,11 +113,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -137,7 +137,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: @@ -145,7 +145,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d2 @@ -165,7 +165,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: a @@ -182,7 +182,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 5500 Data size: 116432 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -200,7 +200,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -220,12 +220,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -242,7 +242,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 2500 Data size: 52560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -253,7 +253,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -273,7 +273,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -287,7 +287,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -306,18 +318,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -325,8 +325,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Vertex 2' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -340,12 +340,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -362,7 +362,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 25000 Data size: 655600 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -373,7 +373,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -390,7 +390,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -404,19 +404,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -428,6 +416,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 4 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -435,7 +435,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -453,12 +453,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -479,7 +479,32 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: d1 @@ -499,7 +524,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -513,7 +538,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 5 Map Operator Tree: TableScan alias: d2 @@ -532,43 +569,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out b/ql/src/test/results/clientpositive/llap/ctas.q.out index e2e7640..b4ca2fb 100644 --- a/ql/src/test/results/clientpositive/llap/ctas.q.out +++ b/ql/src/test/results/clientpositive/llap/ctas.q.out @@ -36,11 +36,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -56,7 +56,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -71,7 +71,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -192,11 +192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -212,7 +212,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -227,7 +227,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -348,11 +348,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -368,7 +368,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -383,7 +383,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -568,11 +568,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -588,7 +588,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -603,7 +603,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -726,11 +726,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -746,7 +746,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -761,7 +761,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/cte_3.q.out b/ql/src/test/results/clientpositive/llap/cte_3.q.out index 27f0ab1..811eaa8 100644 --- a/ql/src/test/results/clientpositive/llap/cte_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -13,15 +13,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -69,15 +61,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -125,27 +109,11 @@ Plan optimized by CBO. Stage-5 Dependency Collection{} Stage-4 - Map 2 llap - File Output Operator [FS_8] - table:{"name:":"default.q1"} - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_9] (rows=1 width=0) - predicate:(key = '5') - TableScan [TS_5] (rows=1 width=0) - default@q2,q2,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 2 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q2"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/cte_5.q.out b/ql/src/test/results/clientpositive/llap/cte_5.q.out index 55afcd4..466348f 100644 --- a/ql/src/test/results/clientpositive/llap/cte_5.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -61,7 +61,7 @@ src_thrift srcbucket srcbucket2 srcpart -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -77,34 +77,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=2 width=89) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=2 width=89) - Conds:(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=1 width=3) - Filter Operator [FIL_11] (rows=1 width=3) - predicate:(UDFToDouble(colnum) = 5.0) - TableScan [TS_0] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out index 081a138..62c24dc 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,30 +14,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=4 width=85) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) - Conds:(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=2 width=85) - Filter Operator [FIL_11] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out index 081a138..62c24dc 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,30 +14,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=4 width=85) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) - Conds:(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=2 width=85) - Filter Operator [FIL_11] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out index cb88f0c..138a0b0 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -13,46 +13,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out index 1c5be08..5ef2789 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -52,46 +52,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -202,46 +173,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out index 0c22b72..2d63e89 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -76,46 +76,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=3) - Conds:RS_11.UDFToDouble(_col0)=RS_12.UDFToDouble(_col0)(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_7] (rows=1 width=3) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=3) - predicate:colnum is not null - TableScan [TS_5] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index ba14835..3faa445 100644 --- a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -105,34 +105,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=5 width=228) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_15] (rows=5 width=228) - Conds:RS_6._col1=RS_7._col3(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=5 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=5 width=88) - predicate:item is not null - TableScan [TS_0] (rows=5 width=88) - default@testdeci2,s,Tbl:COMPLETE,Col:COMPLETE,Output:["id","item"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col3 - Select Operator [SEL_5] (rows=1 width=312) - Output:["_col1","_col2","_col3"] - Filter Operator [FIL_14] (rows=1 width=312) - predicate:((id = 2) and item is not null) - TableScan [TS_3] (rows=1 width=312) - default@testdeci2,d,Tbl:COMPLETE,Col:COMPLETE,Output:["id","amount","sales_tax","item"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index 600a266..b28d5cc 100644 --- a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -97,7 +97,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -200,10 +200,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -222,7 +222,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index d3acbcd..788b781 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -50,10 +50,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -74,7 +74,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -213,11 +213,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -234,7 +234,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -266,43 +300,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -342,11 +342,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -363,27 +363,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -402,7 +382,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -417,6 +397,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -471,11 +471,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -492,7 +492,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -524,43 +558,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 day(_col0) (type: int) - 1 day(_col0) (type: int) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -600,11 +600,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -621,27 +621,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: day(_col0) (type: int) - sort order: + - Map-reduce partition columns: day(_col0) (type: int) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -660,7 +640,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -675,6 +655,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -718,12 +718,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -740,7 +740,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -772,10 +822,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -807,59 +857,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -905,12 +905,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -928,47 +928,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -984,7 +944,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1003,7 +963,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1018,6 +978,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1074,11 +1074,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1094,7 +1094,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -1126,7 +1160,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -1141,43 +1175,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1217,11 +1217,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1238,27 +1238,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1277,7 +1257,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1292,6 +1272,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1344,11 +1344,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1365,7 +1365,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -1397,43 +1431,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1473,11 +1473,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1494,27 +1494,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1533,7 +1513,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1548,6 +1528,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1598,11 +1598,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1619,7 +1619,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1651,43 +1685,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1727,11 +1727,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1748,7 +1748,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (UDFToDouble(_col0) * 2.0) (type: double) + 1 _col0 (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1780,43 +1814,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (UDFToDouble(_col0) * 2.0) (type: double) - 1 _col0 (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1856,11 +1856,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1877,27 +1877,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1916,7 +1896,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1931,6 +1911,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1970,11 +1970,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1991,27 +1991,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2030,7 +2010,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2045,11 +2025,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: ListSink PREHOOK: query: select count(*) from srcpart join srcpart_double_hour on (srcpart.hr*2 = srcpart_double_hour.hr) where srcpart_double_hour.hour = 11 @@ -2097,11 +2097,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2118,7 +2118,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) + 1 UDFToString(_col0) (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -2150,43 +2184,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) - 1 UDFToString(_col0) (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2226,7 +2226,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -2242,12 +2242,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2260,27 +2260,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2299,7 +2279,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2314,7 +2294,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2334,7 +2334,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2361,7 +2361,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2377,11 +2377,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2396,26 +2396,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2440,7 +2421,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2455,6 +2436,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2462,7 +2462,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2497,11 +2497,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2517,7 +2517,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -2549,7 +2583,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -2564,43 +2598,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2642,11 +2642,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2663,42 +2663,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2717,7 +2682,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2732,6 +2697,41 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: ds (string) + Target Input: srcpart + Partition key expr: ds + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Vertex 1 + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2752,11 +2752,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -2788,26 +2788,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2826,7 +2810,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2841,6 +2825,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2863,11 +2863,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2883,7 +2883,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -2915,43 +2949,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2976,12 +2976,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2998,7 +2998,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -3030,10 +3080,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -3053,56 +3103,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3144,12 +3144,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3170,47 +3170,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: (UDFToDouble(hr) = 13.0) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(hr) = 13.0) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3226,7 +3186,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3245,7 +3205,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3260,6 +3220,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: (UDFToDouble(hr) = 13.0) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(hr) = 13.0) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3297,14 +3297,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3320,47 +3322,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3379,7 +3341,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3394,7 +3356,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3412,7 +3394,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3439,8 +3421,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3458,8 +3460,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3497,14 +3497,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3520,47 +3522,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3581,7 +3543,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3596,7 +3558,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3614,7 +3596,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3641,8 +3623,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3660,8 +3662,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3700,52 +3700,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 9 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 9 (CONTAINS) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 9 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3762,27 +3729,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3809,7 +3756,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -3824,8 +3771,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 5 - Reducer 12 + Target Vertex: Vertex 5 + Vertex 11 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3843,7 +3810,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3856,7 +3823,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3874,7 +3841,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ds (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3887,7 +3871,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3905,10 +3909,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -3951,11 +3951,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3972,7 +3972,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3985,7 +3985,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4017,24 +4032,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4089,11 +4089,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4110,7 +4110,7 @@ STAGE PLANS: 0 day(_col0) (type: int) 1 day(_col0) (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4123,7 +4123,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4155,24 +4170,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4216,11 +4216,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4237,7 +4237,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -4246,7 +4246,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4259,7 +4259,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4291,10 +4306,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4326,24 +4341,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4400,11 +4400,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4420,7 +4420,7 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col2 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4433,7 +4433,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -4465,7 +4480,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -4480,24 +4495,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4550,11 +4550,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4571,7 +4571,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4584,7 +4584,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4616,24 +4631,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4675,11 +4675,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4696,7 +4696,7 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4709,7 +4709,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4741,24 +4756,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4798,11 +4798,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4819,7 +4819,7 @@ STAGE PLANS: 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4832,7 +4832,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4864,24 +4879,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4921,7 +4921,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -4939,11 +4939,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4956,7 +4956,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -4976,7 +4976,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4993,7 +4993,7 @@ STAGE PLANS: 0 1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 500000 Data size: 11124000 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5004,7 +5004,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5026,7 +5026,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -5068,11 +5068,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5089,7 +5089,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5102,7 +5102,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5134,24 +5149,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5172,11 +5172,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -5196,7 +5196,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5209,7 +5209,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -5225,21 +5240,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5262,11 +5262,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5282,7 +5282,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5302,7 +5302,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5315,7 +5315,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5354,11 +5354,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5375,7 +5375,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5384,7 +5384,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5397,7 +5397,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5429,10 +5444,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5452,21 +5467,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5508,11 +5508,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5533,7 +5533,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5554,7 +5554,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5563,7 +5563,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1 Data size: 444 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5576,7 +5576,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5596,21 +5611,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5648,13 +5648,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Map 1 (BROADCAST_EDGE), Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Vertex 1 (BROADCAST_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5670,7 +5672,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -5690,27 +5692,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5728,7 +5710,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5744,7 +5726,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -5756,7 +5738,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5771,7 +5753,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5789,8 +5791,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -5866,11 +5866,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_orc @@ -5886,7 +5886,7 @@ STAGE PLANS: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) 1 _col0 (type: string), UDFToDouble(_col2) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 413600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5899,7 +5899,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -5931,7 +5946,7 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: UDFToDouble(_col2) (type: double) outputColumnNames: _col0 @@ -5946,24 +5961,9 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 21d82d2..cfd4a7b 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -156,12 +156,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -179,7 +179,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -199,7 +199,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -232,37 +260,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -329,12 +329,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -352,7 +352,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -372,28 +372,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id (type: int), label (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -407,7 +386,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,6 +400,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: int), label (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -500,7 +500,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) @@ -515,7 +515,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -574,7 +574,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -617,7 +617,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 51 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -628,7 +628,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -651,7 +651,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -702,12 +702,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -725,7 +725,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -745,7 +745,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -778,37 +806,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -867,11 +867,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: agg_01 @@ -889,7 +891,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -900,7 +902,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dim_shops @@ -932,10 +934,10 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: agg_01 @@ -953,7 +955,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -964,7 +966,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: dim_shops @@ -996,11 +998,9 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1059,11 +1059,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1080,7 +1080,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2000000 Data size: 16000000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1093,7 +1093,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1110,21 +1125,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index 2248a35..98ec225 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -168,10 +168,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -191,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -241,11 +241,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -267,7 +267,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -283,7 +283,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -333,10 +333,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -356,7 +356,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -405,10 +405,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -428,7 +428,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -547,10 +547,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -570,7 +570,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -620,11 +620,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -646,7 +646,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -662,7 +662,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -712,10 +712,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -735,7 +735,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -784,10 +784,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -807,7 +807,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1328,10 +1328,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1350,7 +1350,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1400,10 +1400,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1423,7 +1423,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1473,11 +1473,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1494,7 +1494,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1517,7 +1517,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1567,10 +1567,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1590,7 +1590,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1647,10 +1647,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1670,7 +1670,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2047,10 +2047,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2070,7 +2070,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2119,10 +2119,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2142,7 +2142,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2338,10 +2338,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2359,7 +2359,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2556,10 +2556,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2577,7 +2577,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index 5569011..a6a2d3a 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -125,10 +125,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -148,7 +148,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -198,11 +198,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -224,7 +224,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -240,7 +240,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -290,10 +290,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -313,7 +313,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -385,7 +385,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -504,10 +504,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -527,7 +527,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -577,11 +577,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -603,7 +603,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -619,7 +619,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -669,10 +669,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -692,7 +692,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -741,10 +741,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1285,10 +1285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1307,7 +1307,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1357,10 +1357,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1380,7 +1380,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1430,11 +1430,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1451,7 +1451,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1474,7 +1474,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1524,10 +1524,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1547,7 +1547,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1604,10 +1604,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1627,7 +1627,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2004,10 +2004,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2027,7 +2027,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2076,10 +2076,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2099,7 +2099,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2450,10 +2450,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2473,7 +2473,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2524,10 +2524,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2547,7 +2547,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2598,10 +2598,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2621,7 +2621,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2672,10 +2672,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2695,7 +2695,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2746,10 +2746,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2769,7 +2769,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2820,10 +2820,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2843,7 +2843,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2894,7 +2894,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index ba56486..97b9746 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -87,10 +87,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -110,7 +110,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -329,10 +329,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -352,7 +352,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -570,10 +570,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -593,7 +593,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -812,10 +812,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -834,7 +834,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1107,10 +1107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1130,7 +1130,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1349,10 +1349,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1371,7 +1371,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1617,10 +1617,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1643,7 +1643,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1747,10 +1747,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1773,7 +1773,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 604ec61..6a521a7 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -89,10 +89,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -204,7 +204,7 @@ STAGE PLANS: value expressions: _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -373,10 +373,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -395,7 +395,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -465,10 +465,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -487,7 +487,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 347200 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -687,7 +687,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -758,10 +758,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -781,7 +781,7 @@ STAGE PLANS: value expressions: _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -883,10 +883,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -906,7 +906,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1082,10 +1082,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1104,7 +1104,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1175,10 +1175,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1197,7 +1197,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 556800 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1299,10 +1299,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1321,7 +1321,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 422400 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1497,10 +1497,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1520,7 +1520,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1591,10 +1591,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1614,7 +1614,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/empty_join.q.out b/ql/src/test/results/clientpositive/llap/empty_join.q.out index 19aa89f..d927a1f 100644 --- a/ql/src/test/results/clientpositive/llap/empty_join.q.out +++ b/ql/src/test/results/clientpositive/llap/empty_join.q.out @@ -55,40 +55,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Map Join Operator [MAPJOIN_22] (rows=2 width=1) - Conds:SEL_2._col0=RS_10._col0(Left Outer),SEL_2._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col2"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1 width=0) - predicate:id is not null - TableScan [TS_3] (rows=1 width=0) - default@test_2,t2,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_21] (rows=1 width=1) - predicate:id is not null - TableScan [TS_6] (rows=1 width=1) - default@test_3,t3,Tbl:COMPLETE,Col:NONE,Output:["id"] - <-Select Operator [SEL_2] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=1) - predicate:id is not null - TableScan [TS_0] (rows=1 width=1) - default@test_1,t1,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 1 PREHOOK: query: SELECT t1.id, t2.id, t3.id FROM test_1 t1 diff --git a/ql/src/test/results/clientpositive/llap/except_distinct.q.out b/ql/src/test/results/clientpositive/llap/except_distinct.q.out index 94125b5..10136f8 100644 --- a/ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -206,12 +206,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -234,30 +236,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,7 +265,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -309,7 +288,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -338,8 +340,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -368,16 +368,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 14 <- Map 13 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Union 7 (CONTAINS), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) + Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -400,53 +406,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count(1) + aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: src @@ -469,7 +458,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -485,49 +474,43 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: sum(_col2), sum(_col3) + aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 14 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -556,7 +539,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -585,7 +568,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -627,7 +610,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -669,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -692,12 +675,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -726,12 +726,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -752,28 +754,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -807,7 +788,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -830,7 +811,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -864,8 +866,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index bea58fb..9129a98 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -53,13 +53,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.src_orc_merge_test_part"} - Select Operator [SEL_1] (rows=500 width=95) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src PREHOOK: type: QUERY @@ -78,7 +72,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -88,23 +82,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap - File Output Operator [FS_7] - table:{"name:":"default.src_orc_merge_test_part"} - Select Operator [SEL_6] (rows=100 width=95) - Output:["_col0","_col1"] - Limit [LIM_5] (rows=100 width=178) - Number of rows:100 - Select Operator [SEL_4] (rows=100 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Limit [LIM_2] (rows=100 width=178) - Number of rows:100 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' PREHOOK: type: QUERY @@ -123,24 +101,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_2] (rows=500 width=102) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate PREHOOK: type: ALTER_PARTITION_MERGE @@ -157,23 +124,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_2] (rows=500 width=102) - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' PREHOOK: type: QUERY @@ -182,24 +139,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_2] (rows=500 width=102) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table src_orc_merge_test_part PREHOOK: type: DROPTABLE @@ -209,7 +155,7 @@ POSTHOOK: query: drop table src_orc_merge_test_part POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@src_orc_merge_test_part POSTHOOK: Output: default@src_orc_merge_test_part -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( select src1.key as k1, src1.value as v1, @@ -233,42 +179,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_17] - Group By Operator [GBY_15] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(hash(_col0,_col1,_col2,_col3))"] - Select Operator [SEL_11] (rows=27556 width=356) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_20] (rows=27556 width=356) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - Select Operator [SEL_5] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key PREHOOK: type: QUERY @@ -277,27 +196,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_7] - Select Operator [SEL_5] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 2 PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x PREHOOK: type: QUERY @@ -306,37 +211,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_10] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=10 width=101) - Output:["_col0","_col1"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c PREHOOK: type: QUERY @@ -345,98 +227,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_42] - Select Operator [SEL_41] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] - Select Operator [SEL_38] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_37] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_34] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_33] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_52] (rows=3 width=16) - Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_29] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_50] (rows=18 width=84) - predicate:key is not null - TableScan [TS_27] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_51] (rows=1 width=101) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=89) - Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Select Operator [SEL_17] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_16] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=2 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_49] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=2 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_48] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) cbo_t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p left outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int % c asc, cbo_t3.c_int desc PREHOOK: type: QUERY @@ -445,91 +249,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_39] - Select Operator [SEL_38] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - Group By Operator [GBY_34] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] - PartitionCols:_col0, _col1 - Group By Operator [GBY_32] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 - Select Operator [SEL_31] (rows=1 width=16) - Output:["_col6","_col2"] - Filter Operator [FIL_30] (rows=1 width=16) - predicate:(((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_48] (rows=3 width=16) - Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_46] (rows=18 width=84) - predicate:((c_int > 0) and key is not null) - TableScan [TS_24] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_22] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_47] (rows=1 width=101) - Conds:RS_19._col0=RS_20._col0(Left Outer),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_44] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_16] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_45] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c PREHOOK: type: QUERY @@ -538,76 +270,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_31] - Select Operator [SEL_30] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_29] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_27] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_26] (rows=1 width=20) - Output:["_col2","_col6"] - Filter Operator [FIL_25] (rows=1 width=20) - predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) - Merge Join Operator [MERGEJOIN_36] (rows=4 width=20) - Conds:RS_21._col0=RS_22._col0(Right Outer),RS_21._col0=RS_23._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_34] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_16] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_35] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 5 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int PREHOOK: type: QUERY @@ -616,86 +289,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - Group By Operator [GBY_33] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0, _col1 - Group By Operator [GBY_31] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 - Select Operator [SEL_30] (rows=1 width=20) - Output:["_col6","_col2"] - Filter Operator [FIL_29] (rows=1 width=20) - predicate:(((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_42] (rows=4 width=20) - Conds:RS_25._col0=RS_26._col0(Outer),RS_25._col0=RS_27._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=20 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=20 width=84) - predicate:(c_int > 0) - TableScan [TS_22] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_39] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=89) - Output:["_col0","_col1"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Select Operator [SEL_17] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_16] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_40] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 6 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c PREHOOK: type: QUERY @@ -704,83 +310,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_33] - Select Operator [SEL_32] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_31] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Group By Operator [GBY_29] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_28] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_27] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_43] (rows=3 width=16) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=18 width=84) - predicate:key is not null - TableScan [TS_21] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_19] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_42] (rows=1 width=101) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_6] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_39] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_11] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_40] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_8] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 5 PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc PREHOOK: type: QUERY @@ -807,62 +347,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_26] - Select Operator [SEL_25] (rows=3 width=87) - Output:["_col0"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_5] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_1] (rows=20 width=80) - Output:["key"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_12] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_8] (rows=20 width=80) - Output:["key"] - TableScan [TS_7] (rows=20 width=80) - default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_21] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_20] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_17] (rows=20 width=80) - Output:["key"] - TableScan [TS_16] (rows=20 width=80) - default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 UNION ALL @@ -879,76 +373,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 3 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_31] - Select Operator [SEL_30] (rows=1 width=95) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - Group By Operator [GBY_27] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_5] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_1] (rows=20 width=80) - Output:["key"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 7 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_12] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_8] (rows=20 width=80) - Output:["key"] - TableScan [TS_7] (rows=20 width=80) - default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_21] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_20] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_17] (rows=20 width=80) - Output:["key"] - TableScan [TS_16] (rows=20 width=80) - default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 5 PREHOOK: query: explain select cbo_t1.key from cbo_t1 join cbo_t3 where cbo_t1.key=cbo_t3.key and cbo_t1.key >= 1 PREHOOK: type: QUERY @@ -957,34 +392,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=18 width=85) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_13] (rows=6 width=85) - predicate:(UDFToDouble(key) >= 1.0) - TableScan [TS_0] (rows=20 width=80) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_14] (rows=6 width=85) - predicate:(UDFToDouble(key) >= 1.0) - TableScan [TS_3] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 left outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -993,32 +407,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=100 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_11] (rows=100 width=8) - Conds:RS_4._col0=RS_5._col0(Left Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -1027,32 +422,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=100 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_9] (rows=100 width=8) - Conds:RS_4._col0=RS_5._col0(Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select b, cbo_t1.c, cbo_t2.p, q, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1) cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key PREHOOK: type: QUERY @@ -1061,45 +437,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=291 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_24] (rows=291 width=101) - Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=87) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_21] (rows=18 width=87) - predicate:key is not null - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=18 width=84) - predicate:key is not null - TableScan [TS_3] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=18 width=84) - predicate:key is not null - TableScan [TS_6] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select key, cbo_t1.c_int, cbo_t2.p, q from cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.key=p join (select key as a, c_int as b, cbo_t3.c_float as c from cbo_t3)cbo_t3 on cbo_t1.key=a PREHOOK: type: QUERY @@ -1108,45 +452,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=291 width=178) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_24] (rows=291 width=178) - Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_21] (rows=18 width=84) - predicate:key is not null - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_22] (rows=18 width=80) - predicate:key is not null - TableScan [TS_3] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=18 width=84) - predicate:key is not null - TableScan [TS_6] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 full outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) PREHOOK: type: QUERY @@ -1155,55 +467,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_19] - Select Operator [SEL_18] (rows=36 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_17] (rows=36 width=101) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_28] (rows=36 width=101) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=18 width=84) - predicate:key is not null - TableScan [TS_11] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Filter Operator [FIL_9] (rows=10 width=182) - predicate:(((_col1 + _col4) = 2) and ((_col4 + 1) = 2)) - Merge Join Operator [MERGEJOIN_27] (rows=40 width=182) - Conds:RS_6._col0=RS_7._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=9 width=89) - Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) PREHOOK: type: QUERY @@ -1212,45 +483,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=50 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_12] (rows=50 width=101) - predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2)) - Merge Join Operator [MERGEJOIN_19] (rows=200 width=101) - Conds:RS_8._col0=RS_9._col0(Right Outer),RS_8._col0=RS_10._col0(Right Outer),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_17] (rows=10 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=10 width=89) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=10 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_6] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key order by x limit 1 PREHOOK: type: QUERY @@ -1259,34 +498,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Limit [LIM_9] (rows=1 width=97) - Number of rows:1 - Select Operator [SEL_8] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x order by x,y limit 1 PREHOOK: type: QUERY @@ -1295,42 +514,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 4 llap - File Output Operator [FS_15] - Limit [LIM_14] (rows=1 width=20) - Number of rows:1 - Select Operator [SEL_13] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - Group By Operator [GBY_10] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=10 width=101) - Output:["_col0","_col1"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 4 PREHOOK: query: explain select key from(select key from (select key from cbo_t1 limit 5)cbo_t2 limit 5)cbo_t3 limit 5 PREHOOK: type: QUERY @@ -1339,37 +531,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 3 llap - File Output Operator [FS_13] - Limit [LIM_12] (rows=5 width=85) - Number of rows:5 - Limit [LIM_10] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_9] (rows=5 width=85) - Output:["_col0"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Limit [LIM_7] (rows=5 width=85) - Number of rows:5 - Limit [LIM_5] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_4] (rows=5 width=85) - Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Limit [LIM_2] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_1] (rows=20 width=80) - Output:["_col0"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 3 PREHOOK: query: explain select key, c_int from(select key, c_int from (select key, c_int from cbo_t1 order by c_int limit 5)cbo_t1 order by c_int limit 5)cbo_t2 order by c_int limit 5 PREHOOK: type: QUERY @@ -1378,38 +547,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Limit [LIM_12] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_11] (rows=5 width=89) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Limit [LIM_8] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_7] (rows=5 width=89) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Limit [LIM_4] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 4 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5 PREHOOK: type: QUERY @@ -1418,108 +564,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 7 llap - File Output Operator [FS_49] - Limit [LIM_48] (rows=1 width=20) - Number of rows:5 - Select Operator [SEL_47] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - Select Operator [SEL_44] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_43] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0, _col1 - Group By Operator [GBY_41] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_40] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_39] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_61] (rows=3 width=16) - Conds:RS_36._col0=RS_37._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_59] (rows=18 width=84) - predicate:key is not null - TableScan [TS_33] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_60] (rows=2 width=101) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Filter Operator [FIL_26] (rows=2 width=105) - predicate:_col0 is not null - Limit [LIM_24] (rows=3 width=105) - Number of rows:5 - Select Operator [SEL_23] (rows=3 width=105) - Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - Select Operator [SEL_20] (rows=3 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_19] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_17] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_58] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_14] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Filter Operator [FIL_12] (rows=2 width=97) - predicate:_col0 is not null - Limit [LIM_10] (rows=3 width=97) - Number of rows:5 - Select Operator [SEL_9] (rows=3 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=3 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_56] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t1.c_int from cbo_t1 left semi join cbo_t2 on cbo_t1.key=cbo_t2.key where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) PREHOOK: type: QUERY @@ -1528,38 +586,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=9 width=4) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_17] (rows=9 width=4) - Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_16] (rows=18 width=80) - predicate:key is not null - TableScan [TS_3] (rows=20 width=80) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select * from (select c, b, a from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 left semi join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where (b + 1 = 2) and (R.b > 0 or c >= 0) PREHOOK: type: QUERY @@ -1568,49 +601,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=16 width=93) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=16 width=93) - Conds:RS_13._col0=RS_14._col0(Left Semi),RS_13._col0=RS_15._col0(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=3 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=9 width=85) - Output:["_col0"] - Filter Operator [FIL_26] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=6 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_8] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_27] (rows=18 width=80) - predicate:key is not null - TableScan [TS_6] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a PREHOOK: type: QUERY @@ -1619,92 +616,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap - File Output Operator [FS_41] - Select Operator [SEL_40] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - Select Operator [SEL_38] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_37] (rows=1 width=101) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=1 width=101) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 - Merge Join Operator [MERGEJOIN_51] (rows=1 width=93) - Conds:RS_30._col0=RS_31._col0(Left Semi),RS_30._col0=RS_32._col0(Left Semi),Output:["_col0","_col1"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=3 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_25] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_50] (rows=6 width=85) - predicate:(UDFToDouble(key) > 0.0) - TableScan [TS_23] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=93) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - Select Operator [SEL_8] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_7] (rows=1 width=101) - predicate:(((UDFToDouble(_col2) >= 1.0) or (_col3 >= 1)) and ((UDFToDouble(_col2) + UDFToDouble(_col3)) >= 0.0)) - Select Operator [SEL_6] (rows=1 width=101) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_48] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0)) and (UDFToDouble(key) > 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=1 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_21] (rows=1 width=85) - Output:["_col0"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - Select Operator [SEL_18] (rows=1 width=93) - Output:["_col0","_col1"] - Group By Operator [GBY_17] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_15] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_49] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0)) - TableScan [TS_12] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 6 PREHOOK: query: explain select cbo_t1.key as x, c_int as c_int, (((c_int+c_float)*10)+5) as y from cbo_t1 PREHOOK: type: QUERY @@ -1756,33 +680,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_15] - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count('2014')"] - Merge Join Operator [MERGEJOIN_18] (rows=400 width=8) - Conds:(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=20 width=88) - TableScan [TS_0] (rows=20 width=21) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=20 width=88) - TableScan [TS_3] (rows=20 width=21) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE + Vertex 3 PREHOOK: query: explain select * from src_cbo b @@ -1803,71 +708,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_29] - Select Operator [SEL_28] (rows=1 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=1 width=182) - predicate:_col3 is null - Merge Join Operator [MERGEJOIN_37] (rows=500 width=182) - Conds:RS_24._col1=RS_25._col0(Left Outer),Output:["_col0","_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=56 width=95) - Output:["_col0","_col1"] - Group By Operator [GBY_22] (rows=56 width=91) - Output:["_col0"],keys:_col1 - Select Operator [SEL_18] (rows=83 width=178) - Output:["_col1"] - Group By Operator [GBY_17] (rows=83 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=83 width=178) - Output:["_col0","_col1"],keys:_col2, _col0 - Select Operator [SEL_14] (rows=166 width=178) - Output:["_col2","_col0"] - Merge Join Operator [MERGEJOIN_36] (rows=166 width=178) - Conds:RS_11._col1=RS_12._col0(Inner),Output:["_col0","_col2"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_34] (rows=166 width=178) - predicate:(value > 'val_2') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_9] (rows=214 width=91) - Output:["_col0"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=214 width=91) - Output:["_col0"],keys:value - TableScan [TS_5] (rows=500 width=91) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] + Vertex 2 PREHOOK: query: explain select * from src_cbo b @@ -1890,75 +740,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_33] - Select Operator [SEL_32] (rows=1 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_31] (rows=1 width=182) - predicate:_col4 is null - Merge Join Operator [MERGEJOIN_41] (rows=250 width=182) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_4] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1 - Group By Operator [GBY_2] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Select Operator [SEL_27] (rows=1 width=182) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_26] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_40] (rows=1 width=178) - Conds:RS_20._col0, _col1=RS_21._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Select Operator [SEL_8] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_38] (rows=166 width=178) - predicate:(value > 'val_12') - TableScan [TS_6] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=250 width=178) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_13] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_11] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_9] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: create view cv1 as select * @@ -1989,59 +781,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_23] - Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) - Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) - Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_30] (rows=166 width=178) - predicate:(value > 'val_9') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_5] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} + Vertex 2 PREHOOK: query: explain select * from (select * @@ -2064,59 +813,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_23] - Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) - Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) - Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_30] (rows=166 width=178) - predicate:(value > 'val_9') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_5] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from src_cbo @@ -2129,38 +835,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_13] - Merge Join Operator [MERGEJOIN_18] (rows=168 width=178) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=69 width=87) - Output:["_col0"],keys:KEY._col0 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Group By Operator [GBY_5] (rows=69 width=87) - Output:["_col0"],keys:key - Filter Operator [FIL_17] (rows=166 width=87) - predicate:(key > '9') - TableScan [TS_2] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey @@ -2175,109 +857,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_46] - Select Operator [SEL_45] (rows=5 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_67] (rows=5 width=8) - Conds:RS_42._col1, _col4=RS_43._col0, _col1(Inner),Output:["_col0","_col3"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col1, _col4 - Merge Join Operator [MERGEJOIN_64] (rows=5 width=16) - Conds:RS_39._col0=RS_40._col1(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=17 width=16) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_60] (rows=17 width=16) - predicate:((l_linenumber = 1) and l_partkey is not null) - TableScan [TS_7] (rows=100 width=16) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_5] (rows=50 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Group By Operator [GBY_3] (rows=50 width=4) - Output:["_col0"],keys:l_partkey - Filter Operator [FIL_59] (rows=100 width=4) - predicate:l_partkey is not null - TableScan [TS_0] (rows=100 width=4) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_37] (rows=4 width=8) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=4 width=8) - Output:["_col0","_col1"],keys:_col0, _col3 - Merge Join Operator [MERGEJOIN_66] (rows=14 width=8) - Conds:RS_31._col1=RS_32._col0(Inner),Output:["_col0","_col3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=14 width=95) - Output:["_col0","_col1"] - Filter Operator [FIL_61] (rows=14 width=96) - predicate:(l_shipmode = 'AIR') - TableScan [TS_10] (rows=100 width=96) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=3 width=4) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=3 width=4) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_65] (rows=34 width=4) - Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col2"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=100 width=8) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=100 width=8) - predicate:l_partkey is not null - TableScan [TS_20] (rows=100 width=8) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey","l_linenumber"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Group By Operator [GBY_18] (rows=50 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=50 width=4) - Output:["_col0"],keys:l_partkey - Filter Operator [FIL_62] (rows=100 width=4) - predicate:l_partkey is not null - TableScan [TS_13] (rows=100 width=4) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] + Vertex 4 PREHOOK: query: explain select key, value, count(*) from src_cbo b @@ -2294,77 +887,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_33] - Merge Join Operator [MERGEJOIN_46] (rows=34 width=186) - Conds:RS_29._col2=RS_30._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col2 - Group By Operator [GBY_15] (rows=84 width=186) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0, _col1 - Group By Operator [GBY_13] (rows=84 width=186) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 - Merge Join Operator [MERGEJOIN_45] (rows=168 width=178) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=69 width=87) - Output:["_col0"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Group By Operator [GBY_5] (rows=69 width=87) - Output:["_col0"],keys:key - Filter Operator [FIL_41] (rows=166 width=87) - predicate:(key > '8') - TableScan [TS_2] (rows=500 width=87) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=34 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=34 width=8) - Output:["_col0"],keys:_col1 - Select Operator [SEL_44] (rows=69 width=8) - Output:["_col1"] - Group By Operator [GBY_22] (rows=69 width=95) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_20] (rows=69 width=95) - Output:["_col0","_col1"],aggregations:["count()"],keys:key - Filter Operator [FIL_43] (rows=166 width=87) - predicate:(key > '9') - TableScan [TS_17] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, avg(p_size) from part @@ -2381,54 +915,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_21] - Merge Join Operator [MERGEJOIN_26] (rows=6 width=227) - Conds:RS_17._col1=RS_18._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=13 width=227) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_4] (rows=13 width=227) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1 - Group By Operator [GBY_2] (rows=13 width=295) - Output:["_col0","_col1","_col2"],aggregations:["avg(p_size)"],keys:p_name, p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=13 width=184) - Output:["_col0"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=13 width=184) - Output:["_col0"],keys:_col0 - Select Operator [SEL_10] (rows=26 width=491) - Output:["_col0"] - PTF Operator [PTF_9] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_8] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:p_mfgr - TableScan [TS_6] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 3 PREHOOK: query: explain select * from src_cbo @@ -2447,67 +943,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_27] - Select Operator [SEL_26] (rows=500 width=178) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - Select Operator [SEL_24] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=500 width=198) - predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) - Merge Join Operator [MERGEJOIN_32] (rows=500 width=198) - Conds:RS_20._col0=RS_21._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_31] (rows=500 width=194) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - Group By Operator [GBY_7] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(key)"] - Filter Operator [FIL_29] (rows=166 width=87) - predicate:(key > '2') - TableScan [TS_2] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=69 width=91) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0, _col1 - Group By Operator [GBY_13] (rows=69 width=91) - Output:["_col0","_col1"],keys:_col0, true - Select Operator [SEL_11] (rows=166 width=87) - Output:["_col0"] - Filter Operator [FIL_30] (rows=166 width=87) - predicate:(key > '2') - TableScan [TS_9] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select p_mfgr, b.p_name, p_size from part b @@ -2528,128 +974,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_54] - Select Operator [SEL_53] (rows=13 width=223) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_52] (rows=13 width=243) - predicate:CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END - Merge Join Operator [MERGEJOIN_76] (rows=26 width=243) - Conds:RS_49._col0, _col1=RS_50._col3, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] - PartitionCols:_col3, _col1 - Merge Join Operator [MERGEJOIN_75] (rows=2 width=223) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=4 width=223) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_34] (rows=4 width=219) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] - PartitionCols:_col0, _col1 - Group By Operator [GBY_32] (rows=4 width=219) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_74] (rows=8 width=219) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=8 width=219) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=8 width=223) - predicate:(p_size < 10) - TableScan [TS_19] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_22] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] - PartitionCols:_col0 - Group By Operator [GBY_40] (rows=13 width=121) - Output:["_col0"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=13 width=121) - Output:["_col0"],keys:p_name - TableScan [TS_36] (rows=26 width=121) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_73] (rows=26 width=239) - Conds:RS_46._col1=RS_47._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - PartitionCols:_col1 - Select Operator [SEL_1] (rows=26 width=223) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=26 width=223) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=2 width=114) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=2 width=114) - Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col1)"],keys:_col2 - Select Operator [SEL_14] (rows=8 width=219) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_72] (rows=8 width=219) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col1","_col2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_4] (rows=8 width=219) - Output:["_col0","_col1"] - Filter Operator [FIL_67] (rows=8 width=223) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_9] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_5] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] + Vertex 3 PREHOOK: query: explain select p_name, p_size from @@ -2670,74 +1010,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_36] - Select Operator [SEL_35] (rows=26 width=125) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=26 width=125) - Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=26 width=145) - predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) - Merge Join Operator [MERGEJOIN_42] (rows=26 width=145) - Conds:RS_29.UDFToDouble(_col1)=RS_30._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:UDFToDouble(_col1) - Merge Join Operator [MERGEJOIN_41] (rows=26 width=141) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_26] - Select Operator [SEL_1] (rows=26 width=125) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=26 width=125) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] - Group By Operator [GBY_12] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] - Group By Operator [GBY_7] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_38] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=1 width=12) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0, _col1 - Group By Operator [GBY_22] (rows=1 width=12) - Output:["_col0","_col1"],keys:_col0, true - Group By Operator [GBY_19] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_40] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_14] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] + Vertex 4 PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) from part b @@ -2762,185 +1046,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_83] - Select Operator [SEL_82] (rows=2 width=106) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_81] - Select Operator [SEL_80] (rows=2 width=106) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=2 width=126) - predicate:CASE WHEN ((_col3 = 0)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END - Merge Join Operator [MERGEJOIN_108] (rows=5 width=126) - Conds:RS_76._col0, _col1=RS_77._col3, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_77] - PartitionCols:_col3, _col1 - Merge Join Operator [MERGEJOIN_107] (rows=1 width=110) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=1 width=110) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_56] (rows=1 width=106) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col0, _col1 - Group By Operator [GBY_54] (rows=1 width=106) - Output:["_col0","_col1"],keys:_col0, _col3 - Merge Join Operator [MERGEJOIN_106] (rows=1 width=106) - Conds:RS_50._col1=RS_51._col0(Inner),Output:["_col0","_col3"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=1 width=114) - Output:["_col0","_col1"] - Filter Operator [FIL_98] (rows=1 width=114) - predicate:((_col2 - _col1) > 600.0) - Group By Operator [GBY_36] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - PartitionCols:_col0 - Group By Operator [GBY_34] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - TableScan [TS_32] (rows=26 width=106) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 20 [SIMPLE_EDGE] llap - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_48] (rows=2 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 19 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_46] (rows=2 width=8) - Output:["_col0"],keys:_col1 - Select Operator [SEL_103] (rows=5 width=8) - Output:["_col1"] - Group By Operator [GBY_43] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_40] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_39] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_70] - PartitionCols:_col0 - Group By Operator [GBY_67] (rows=5 width=98) - Output:["_col0"],keys:_col0 - Group By Operator [GBY_62] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_61] - PartitionCols:_col0 - Group By Operator [GBY_60] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_58] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_76] - PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_105] (rows=5 width=122) - Conds:RS_73._col1=RS_74._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_73] - PartitionCols:_col1 - Group By Operator [GBY_4] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0 - Group By Operator [GBY_2] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_1] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_0] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_74] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col3 - Select Operator [SEL_27] (rows=1 width=106) - Output:["_col3","_col0"] - Merge Join Operator [MERGEJOIN_104] (rows=1 width=106) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col0","_col3"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_22] (rows=2 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_20] (rows=2 width=8) - Output:["_col0"],keys:_col1 - Select Operator [SEL_102] (rows=5 width=8) - Output:["_col1"] - Group By Operator [GBY_17] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_14] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_13] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=1 width=114) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=1 width=114) - predicate:((_col2 - _col1) > 600.0) - Group By Operator [GBY_10] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_7] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_6] (rows=26 width=106) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] + Vertex 5 PREHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 PREHOOK: type: QUERY @@ -2949,25 +1076,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=52) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - PTF Operator [PTF_3] (rows=20 width=459) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_2] (rows=20 width=459) - Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:0 - TableScan [TS_0] (rows=20 width=7) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + Vertex 2 PREHOOK: query: explain select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1 PREHOOK: type: QUERY @@ -2976,25 +1091,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=52) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - PTF Operator [PTF_3] (rows=20 width=459) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_2] (rows=20 width=459) - Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:0 - TableScan [TS_0] (rows=20 width=7) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + Vertex 2 PREHOOK: query: explain select i, a, h, b, c, d, e, f, g, a as x, a +1 as y from (select max(c_int) over (partition by key order by value range UNBOUNDED PRECEDING) a, min(c_int) over (partition by key order by value range current row) b, count(c_int) over(partition by key order by value range 1 PRECEDING) c, avg(value) over (partition by key order by value range between unbounded preceding and unbounded following) d, sum(value) over (partition by key order by value range between unbounded preceding and current row) e, avg(c_float) over (partition by key order by value range between 1 preceding and unbounded following) f, sum(c_float) over (partition by key order by value range between 1 preceding and current row) g, max(c_float) over (partition by key order by value range between 1 preceding and unbounded following) h, min(c_float) over (partition by key order by value range between 1 preceding and 1 following) i from cbo_t1) cbo_t1 PREHOOK: type: QUERY @@ -3003,25 +1106,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=64) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - PTF Operator [PTF_3] (rows=20 width=621) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_2] (rows=20 width=621) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:key - TableScan [TS_0] (rows=20 width=169) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int","key","value"] + Vertex 2 PREHOOK: query: explain select *, rank() over(partition by key order by value) as rr from src1 PREHOOK: type: QUERY @@ -3030,25 +1121,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=25 width=179) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_3] (rows=25 width=443) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_2] (rows=25 width=443) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:key - TableScan [TS_0] (rows=25 width=175) - default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3065,51 +1144,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3126,51 +1169,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3187,48 +1194,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Map Join Operator [MAPJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3245,53 +1219,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_22] - Group By Operator [GBY_20] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_16] (rows=12 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_15] (rows=12 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=12 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=25 width=86) - Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_25] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=205 width=87) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_26] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain create table abcd (a int, b int, c int, d int) PREHOOK: type: CREATETABLE @@ -3324,25 +1260,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=2 width=19) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col2)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0 - Group By Operator [GBY_2] (rows=4 width=19) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(DISTINCT b)","count(DISTINCT c)","sum(d)"],keys:a, b, c - Select Operator [SEL_1] (rows=4 width=19) - Output:["a","b","c","d"] - TableScan [TS_0] (rows=4 width=19) - default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + Vertex 2 PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a PREHOOK: type: QUERY @@ -3351,23 +1275,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=2 width=19) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:a - Select Operator [SEL_1] (rows=4 width=19) - Output:["a","b","c","d"] - TableScan [TS_0] (rows=4 width=19) - default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + Vertex 2 PREHOOK: query: explain create table src_rc_merge_test(key int, value string) stored as rcfile PREHOOK: type: CREATETABLE @@ -3443,21 +1357,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test PREHOOK: type: QUERY @@ -3466,22 +1372,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: alter table tgt_rc_merge_test concatenate PREHOOK: type: ALTER_TABLE_MERGE @@ -3515,21 +1412,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test PREHOOK: type: QUERY @@ -3538,22 +1427,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table src_rc_merge_test PREHOOK: type: DROPTABLE @@ -3578,27 +1458,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Merge Join Operator [MERGEJOIN_9] (rows=250000 width=87) - Conds:(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - Select Operator [SEL_1] (rows=500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Select Operator [SEL_3] (rows=500 width=4) - TableScan [TS_2] (rows=500 width=10) - default@src,src2,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain create table nzhang_Tmp(a int, b string) PREHOOK: type: CREATETABLE @@ -3623,8 +1489,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3634,25 +1500,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap - File Output Operator [FS_8] - table:{"name:":"default.nzhang_CTAS1"} - Limit [LIM_7] (rows=10 width=178) - Number of rows:10 - Select Operator [SEL_6] (rows=10 width=178) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Limit [LIM_4] (rows=10 width=178) - Number of rows:10 - Select Operator [SEL_3] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -3676,8 +1524,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3687,25 +1535,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap - File Output Operator [FS_8] - table:{"name:":"default.nzhang_ctas3"} - Limit [LIM_7] (rows=10 width=192) - Number of rows:10 - Select Operator [SEL_6] (rows=10 width=192) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Limit [LIM_4] (rows=10 width=192) - Number of rows:10 - Select Operator [SEL_3] (rows=500 width=192) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=192) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -3766,37 +1596,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=27556 width=356) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_15] (rows=27556 width=356) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: CREATE TABLE myinput1(key int, value int) PREHOOK: type: CREATETABLE @@ -3821,28 +1628,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key PREHOOK: type: QUERY @@ -3851,39 +1643,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Select Operator [SEL_10] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_21] (rows=6 width=9) - Conds:RS_4.key=RS_6.value(Inner),RS_4.key=RS_8.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key - Filter Operator [FIL_18] (rows=3 width=8) - predicate:key is not null - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:value - Filter Operator [FIL_19] (rows=3 width=8) - predicate:value is not null - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:key - Filter Operator [FIL_20] (rows=3 width=8) - predicate:key is not null - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key PREHOOK: type: QUERY @@ -3892,33 +1658,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) - Conds:RS_3.key=RS_4.value(Inner),RS_3.key=RS_5.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value PREHOOK: type: QUERY @@ -3927,39 +1673,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Select Operator [SEL_10] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_15] (rows=6 width=9) - Conds:RS_4.key, value=RS_6.value, key(Inner),RS_4.key, value=RS_8.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key, value - Filter Operator [FIL_12] (rows=3 width=8) - predicate:value is not null - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:value, key - Filter Operator [FIL_13] (rows=3 width=8) - predicate:key is not null - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:key, value - Filter Operator [FIL_14] (rows=3 width=8) - predicate:value is not null - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value PREHOOK: type: QUERY @@ -3968,33 +1688,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) - Conds:RS_3.key, value=RS_4.value, key(Inner),RS_3.key, value=RS_5.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key, value - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:value, key - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key, value - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4003,28 +1703,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4033,28 +1718,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4063,28 +1733,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4093,28 +1748,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: CREATE TABLE smb_input(key int, value int) PREHOOK: type: CREATETABLE @@ -4189,28 +1829,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key AND a.value <=> b.value PREHOOK: type: QUERY @@ -4219,28 +1844,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key, value=RS_3.key, value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key, value - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key, value - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4249,28 +1859,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4279,28 +1874,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4309,28 +1889,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table sales PREHOOK: type: DROPTABLE @@ -4393,36 +1958,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=15) - Conds:RS_8._col1=RS_9._col0(Left Semi),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1 width=13) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=1 width=13) - predicate:id is not null - TableScan [TS_0] (rows=1 width=13) - default@sales,sales,Tbl:COMPLETE,Col:NONE,Output:["name","id"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=2 width=14) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=2 width=14) - Output:["_col0"] - Filter Operator [FIL_16] (rows=2 width=14) - predicate:id is not null - TableScan [TS_3] (rows=2 width=14) - default@things,things,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: drop table sales PREHOOK: type: DROPTABLE @@ -4447,42 +1989,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Map Join Operator [MAPJOIN_26] (rows=805 width=10) - Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=166 width=10) - Output:["_col0"] - Filter Operator [FIL_24] (rows=166 width=10) - predicate:(value > 'val_450') - TableScan [TS_6] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] - <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) - Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:key is not null - TableScan [TS_3] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=666 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=666 width=10) - predicate:((value > 'val_450') and key is not null) - TableScan [TS_0] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' PREHOOK: type: QUERY @@ -4491,42 +2004,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Map Join Operator [MAPJOIN_26] (rows=805 width=10) - Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=166 width=10) - Output:["_col0"] - Filter Operator [FIL_24] (rows=166 width=10) - predicate:(value > 'val_450') - TableScan [TS_6] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] - <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) - Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:key is not null - TableScan [TS_3] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=666 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=666 width=10) - predicate:((value > 'val_450') and key is not null) - TableScan [TS_0] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4551,33 +2035,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, @@ -4596,45 +2061,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Select Operator [SEL_14] (rows=29 width=227) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_13] (rows=29 width=223) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_12] (rows=29 width=223) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - PTF Operator [PTF_10] (rows=29 width=223) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_9] (rows=29 width=223) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_21] (rows=29 width=223) - Conds:FIL_19.p_partkey=RS_5.p_partkey(Inner),Output:["_col1","_col2","_col5"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:p_partkey - Filter Operator [FIL_20] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p2,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_19] (rows=26 width=227) - predicate:p_partkey is not null - TableScan [TS_0] (rows=26 width=227) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey","p_name","p_mfgr","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4659,33 +2094,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4710,33 +2126,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4763,41 +2160,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_14] - Select Operator [SEL_12] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_11] (rows=26 width=223) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_8] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:rand() - Select Operator [SEL_4] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select abc.* @@ -4816,34 +2187,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Map Join Operator [MAPJOIN_16] (rows=29 width=619) - Conds:FIL_14._col0=RS_8.p_partkey(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_8] - PartitionCols:p_partkey - Filter Operator [FIL_15] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_14] (rows=26 width=887) - predicate:_col0 is not null - PTF Operator [PTF_4] (rows=26 width=887) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=887) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=619) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_brand","p_comment","p_container","p_mfgr","p_name","p_partkey","p_retailprice","p_size","p_type"] + Vertex 2 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4862,35 +2212,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_8] (rows=26 width=227) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_7] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}] - Select Operator [SEL_6] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - PTF Operator [PTF_1] (rows=26 width=223) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST, p_size DESC NULLS LAST","partition by:":"p_mfgr"}}] - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4913,35 +2242,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_8] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_7] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_6] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - PTF Operator [PTF_1] (rows=26 width=231) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST","partition by:":"p_mfgr"}}] - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size","p_retailprice"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4964,33 +2272,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -5015,43 +2304,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Select Operator [SEL_11] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_10] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_9] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2 - PTF Operator [PTF_7] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_6] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, @@ -5082,33 +2343,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_7] (rows=26 width=235) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select abc.p_mfgr, abc.p_name, @@ -5137,44 +2379,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_15] - Select Operator [SEL_13] (rows=29 width=259) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - PTF Operator [PTF_12] (rows=29 width=767) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=29 width=767) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_20] (rows=29 width=231) - Conds:FIL_18._col0=RS_8.p_partkey(Inner),Output:["_col1","_col2","_col5","_col7"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_8] - PartitionCols:p_partkey - Filter Operator [FIL_19] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_18] (rows=26 width=503) - predicate:_col0 is not null - PTF Operator [PTF_4] (rows=26 width=503) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=503) - Output:["_col0","_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=235) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_partkey","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain create view IF NOT EXISTS mfgr_price_view as select p_mfgr, p_brand, @@ -5276,10 +2488,10 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -5289,49 +2501,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 3 llap - File Output Operator [FS_9] - table:{"name:":"default.part_4"} - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] - Reducer 5 llap - File Output Operator [FS_20] - table:{"name:":"default.part_5"} - Select Operator [SEL_17] (rows=26 width=247) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - PTF Operator [PTF_16] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_15] (rows=26 width=499) - Output:["_col0","_col2","_col3","_col6"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col2 - Select Operator [SEL_13] (rows=26 width=491) - Output:["_col1","_col2","_col5","sum_window_0"] - PTF Operator [PTF_12] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - Please refer to the previous PTF Operator [PTF_3] + Vertex 3 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -5376,43 +2547,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Select Operator [SEL_11] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_10] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}] - Select Operator [SEL_9] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2, _col1 - PTF Operator [PTF_7] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_6] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2, _col1 - PTF Operator [PTF_4] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, @@ -5451,49 +2594,16 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_15] - Select Operator [SEL_13] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_12] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - PTF Operator [PTF_9] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_8] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2, _col1 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 5 PREHOOK: query: explain select p_mfgr, p_name, @@ -5528,41 +2638,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_12] - Select Operator [SEL_10] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_9] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_8] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr, p_name - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select distinct src.* from src PREHOOK: type: QUERY @@ -5571,29 +2655,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_7] - Group By Operator [GBY_5] (rows=500 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1 - Group By Operator [GBY_3] (rows=500 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:rand() - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: explain select explode(array('a', 'b')) PREHOOK: type: QUERY @@ -5693,7 +2762,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -5703,31 +2772,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - table:{"name:":"default.dest_j1"} - Select Operator [SEL_9] (rows=1219 width=95) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_16] (rows=1219 width=178) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_14] (rows=500 width=87) - predicate:key is not null - TableScan [TS_0] (rows=500 width=87) - default@src,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,src2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value @@ -5756,43 +2801,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=3 width=33) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Map Join Operator [MAPJOIN_31] (rows=3 width=33) - Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:key - Filter Operator [FIL_28] (rows=1 width=30) - predicate:key is not null - TableScan [TS_1] (rows=1 width=30) - default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_9] - PartitionCols:key - Filter Operator [FIL_29] (rows=1 width=20) - predicate:key is not null - TableScan [TS_2] (rows=1 width=20) - default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:key - Filter Operator [FIL_30] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_27] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 1 PREHOOK: query: explain select /*+ STREAMTABLE(a,c) */ * @@ -5809,43 +2824,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=3 width=33) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Map Join Operator [MAPJOIN_31] (rows=3 width=33) - Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:key - Filter Operator [FIL_28] (rows=1 width=30) - predicate:key is not null - TableScan [TS_1] (rows=1 width=30) - default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_9] - PartitionCols:key - Filter Operator [FIL_29] (rows=1 width=20) - predicate:key is not null - TableScan [TS_2] (rows=1 width=20) - default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:key - Filter Operator [FIL_30] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_27] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 1 PREHOOK: query: explain FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -5854,38 +2839,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:rand() - Map Join Operator [MAPJOIN_18] (rows=550 width=87) - Conds:RS_3.UDFToDouble(key)=FIL_17.(key + 1)(Inner),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_3] - PartitionCols:UDFToDouble(key) - Filter Operator [FIL_16] (rows=1 width=30) - predicate:UDFToDouble(key) is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_17] (rows=500 width=87) - predicate:(key + 1) is not null - TableScan [TS_1] (rows=500 width=87) - default@src,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -5915,34 +2877,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=1219 width=356) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val PREHOOK: type: QUERY @@ -5951,38 +2892,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:rand() - Map Join Operator [MAPJOIN_18] (rows=1 width=33) - Conds:FIL_16.key=RS_5.val(Inner),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:val - Filter Operator [FIL_17] (rows=1 width=30) - predicate:val is not null - TableScan [TS_1] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["val"] - <-Filter Operator [FIL_16] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key PREHOOK: type: QUERY @@ -5991,44 +2909,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Group By Operator [GBY_14] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:rand() - Select Operator [SEL_9] (rows=1 width=33) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_21] (rows=1 width=33) - Conds:SEL_2._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=30) - Output:["_col0","_col1"] - Filter Operator [FIL_20] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Select Operator [SEL_2] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain select count(1) from T1 a join T1 b on a.key = b.key PREHOOK: type: QUERY @@ -6037,42 +2926,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Group By Operator [GBY_14] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:rand() - Map Join Operator [MAPJOIN_21] (rows=1 width=33) - Conds:SEL_2._col0=RS_7._col0(Inner) - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key select sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6081,40 +2943,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=33) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_17] (rows=1 width=33) - Conds:SEL_1.UDFToDouble(_col0)=RS_5.(UDFToDouble(_col0) + 1.0)(Left Outer),Output:["_col0","_col1","_col2"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:(UDFToDouble(_col0) + 1.0) - Select Operator [SEL_3] (rows=1 width=30) - Output:["_col0"] - TableScan [TS_2] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_1] (rows=1 width=30) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 3 PREHOOK: query: explain FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6123,34 +2960,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:rand() - Map Join Operator [MAPJOIN_14] (rows=1 width=33) - Conds:RS_2.UDFToDouble(key)=TS_1.(key + 1)(Right Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_2] - PartitionCols:UDFToDouble(key) - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-TableScan [TS_1] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 4 PREHOOK: query: explain FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6159,37 +2977,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:rand() - Merge Join Operator [MERGEJOIN_12] (rows=1 width=33) - Conds:RS_2.UDFToDouble(key)=RS_3.(key + 1)(Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:UDFToDouble(key) - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:(key + 1) - TableScan [TS_1] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 4 PREHOOK: query: explain select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key PREHOOK: type: QUERY @@ -6198,32 +2994,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:rand() - Map Join Operator [MAPJOIN_14] (rows=1 width=33) - Conds:TS_0.(key + 1)=RS_3.UDFToDouble(key)(Left Outer),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_3] - PartitionCols:UDFToDouble(key) - TableScan [TS_1] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 9bb478c..b82cf89 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -181,51 +181,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=141 width=268) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_26] (rows=141 width=268) - Conds:RS_12._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=500 width=178) - predicate:key is not null - TableScan [TS_6] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_25] (rows=58 width=177) - Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=369) - Output:["_col0"] - Filter Operator [FIL_22] (rows=500 width=91) - predicate:value is not null - TableScan [TS_0] (rows=500 width=91) - default@srcpart,z,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: EXPLAIN select @@ -286,143 +249,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap - File Output Operator [FS_55] - Limit [LIM_54] (rows=24 width=285) - Number of rows:100 - Select Operator [SEL_53] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] - Group By Operator [GBY_50] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_48] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 - Select Operator [SEL_47] (rows=650 width=534) - Output:["_col12","_col20","_col2","_col13","_col21","_col3"] - Merge Join Operator [MERGEJOIN_97] (rows=650 width=534) - Conds:RS_44._col1, _col3=RS_45._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] - PartitionCols:_col15, _col17 - Select Operator [SEL_40] (rows=190 width=447) - Output:["_col14","_col15","_col17","_col6","_col7"] - Merge Join Operator [MERGEJOIN_96] (rows=190 width=447) - Conds:RS_37._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - PartitionCols:_col6, _col4 - Merge Join Operator [MERGEJOIN_94] (rows=40 width=352) - Conds:RS_34._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - PartitionCols:_col1 - Select Operator [SEL_17] (rows=2 width=180) - Output:["_col1"] - Filter Operator [FIL_88] (rows=2 width=175) - predicate:((key = 'src1key') and value is not null) - TableScan [TS_15] (rows=25 width=175) - default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_93] (rows=40 width=352) - Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=2 width=178) - Output:["_col0"] - Filter Operator [FIL_87] (rows=2 width=178) - predicate:((value = 'd1value') and key is not null) - TableScan [TS_12] (rows=500 width=178) - default@src,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_92] (rows=40 width=352) - Conds:RS_28._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col3 - Select Operator [SEL_11] (rows=8 width=531) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_86] (rows=8 width=534) - predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null) - TableScan [TS_9] (rows=85 width=534) - default@ss,ss,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=10 width=185) - Output:["_col1"] - Filter Operator [FIL_85] (rows=10 width=178) - predicate:((key = 'srcpartkey') and value is not null) - TableScan [TS_6] (rows=2000 width=178) - default@srcpart,srcpart,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_38] - PartitionCols:_col4, _col2 - Merge Join Operator [MERGEJOIN_95] (rows=19 width=356) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=8 width=531) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_89] (rows=8 width=534) - predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_18] (rows=85 width=534) - default@sr,sr,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500 width=178) - Output:["_col0"] - Filter Operator [FIL_90] (rows=500 width=178) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_21] (rows=500 width=178) - default@src,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_44] - PartitionCols:_col1, _col3 - Merge Join Operator [MERGEJOIN_91] (rows=414 width=269) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=170 width=356) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_83] (rows=170 width=356) - predicate:(v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_0] (rows=170 width=356) - default@cs,cs,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v2","k3","v3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0"] - Filter Operator [FIL_84] (rows=500 width=178) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,d3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, z.value, y.value @@ -445,151 +286,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_59] - Group By Operator [GBY_57] (rows=28 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=28 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_51] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_85] (rows=73 width=177) - Conds:RS_48._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_81] (rows=500 width=87) - predicate:key is not null - TableScan [TS_42] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_48] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_84] (rows=30 width=177) - Conds:RS_45._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_39] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_37] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_78] (rows=25 width=175) - predicate:value is not null - TableScan [TS_26] (rows=25 width=175) - Output:["key","value"] - <-Map 16 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=500 width=178) - predicate:value is not null - TableScan [TS_29] (rows=500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=28 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_83] (rows=73 width=177) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_77] (rows=500 width=87) - predicate:key is not null - TableScan [TS_16] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_82] (rows=30 width=177) - Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25 width=175) - Output:["key","value"] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - Output:["key","value"] + Vertex 7 PREHOOK: query: explain SELECT x.key, y.value @@ -620,296 +333,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=107 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=107 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=124 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_170] (rows=124 width=177) - Conds:RS_111._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] llap - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=178) - predicate:key is not null - TableScan [TS_105] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] llap - SHUFFLE [RS_111] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_169] (rows=51 width=86) - Conds:RS_108._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] llap - SHUFFLE [RS_109] - PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_102] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] llap - SHUFFLE [RS_108] - PartitionCols:_col1 - Select Operator [SEL_101] (rows=440 width=178) - Output:["_col1"] - Group By Operator [GBY_100] (rows=440 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=440 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=178) - predicate:value is not null - TableScan [TS_92] (rows=500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=440 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_91] (rows=381 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_90] (rows=381 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=178) - predicate:value is not null - TableScan [TS_82] (rows=500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_81] (rows=262 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_80] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_71] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=25 width=175) - predicate:value is not null - TableScan [TS_69] (rows=25 width=175) - Output:["key","value"] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_74] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=178) - predicate:value is not null - TableScan [TS_72] (rows=500 width=178) - Output:["key","value"] - <-Reducer 7 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=107 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=90 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=90 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=107 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_168] (rows=107 width=177) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=178) - predicate:key is not null - TableScan [TS_52] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_167] (rows=44 width=86) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=178) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=178) - predicate:value is not null - TableScan [TS_39] (rows=500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=175) - predicate:value is not null - TableScan [TS_26] (rows=25 width=175) - Output:["key","value"] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=178) - predicate:value is not null - TableScan [TS_29] (rows=500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=90 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_166] (rows=73 width=177) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=178) - predicate:key is not null - TableScan [TS_16] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_165] (rows=30 width=86) - Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25 width=175) - Output:["key","value"] - <-Map 10 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - Output:["key","value"] + Vertex 9 PREHOOK: query: EXPLAIN SELECT x.key, z.value, y.value @@ -924,44 +376,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=605 width=10) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_26] (rows=605 width=10) - Conds:MAPJOIN_25._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=500 width=10) - predicate:key is not null - TableScan [TS_6] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_25] (rows=550 width=10) - Conds:SEL_2._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_22] (rows=500 width=10) - predicate:value is not null - TableScan [TS_0] (rows=500 width=10) - default@srcpart,z,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 1 PREHOOK: query: EXPLAIN select @@ -1022,118 +443,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 10 <- Map 9 (BROADCAST_EDGE) -Map 2 <- Map 1 (BROADCAST_EDGE) -Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 10 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap - File Output Operator [FS_55] - Limit [LIM_54] (rows=100 width=10) - Number of rows:100 - Select Operator [SEL_53] (rows=805 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] - Group By Operator [GBY_50] (rows=805 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_48] (rows=1610 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 - Select Operator [SEL_47] (rows=1610 width=10) - Output:["_col12","_col20","_col2","_col13","_col21","_col3"] - Map Join Operator [MAPJOIN_97] (rows=1610 width=10) - Conds:RS_44._col1, _col3=SEL_40._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_44] - PartitionCols:_col1, _col3 - Map Join Operator [MAPJOIN_91] (rows=275 width=10) - Conds:RS_41._col0=SEL_5._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=170 width=34) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_83] (rows=170 width=34) - predicate:(v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_0] (rows=170 width=34) - default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"] - <-Select Operator [SEL_5] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_84] (rows=250 width=10) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_3] (rows=500 width=10) - default@src,d3,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_40] (rows=1464 width=10) - Output:["_col14","_col15","_col17","_col6","_col7"] - Map Join Operator [MAPJOIN_96] (rows=1464 width=10) - Conds:MAPJOIN_94._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_38] - PartitionCols:_col4, _col2 - Map Join Operator [MAPJOIN_95] (rows=275 width=10) - Conds:RS_24._col0=SEL_23._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 9 [BROADCAST_EDGE] llap - BROADCAST [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=42 width=34) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_89] (rows=42 width=34) - predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_18] (rows=85 width=34) - default@sr,sr,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"] - <-Select Operator [SEL_23] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_90] (rows=250 width=10) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_21] (rows=500 width=10) - default@src,d2,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_94] (rows=1331 width=10) - Conds:MAPJOIN_93._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 8 [BROADCAST_EDGE] llap - BROADCAST [RS_35] - PartitionCols:_col1 - Select Operator [SEL_17] (rows=12 width=7) - Output:["_col1"] - Filter Operator [FIL_88] (rows=12 width=7) - predicate:((key = 'src1key') and value is not null) - TableScan [TS_15] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_93] (rows=1210 width=10) - Conds:MAPJOIN_92._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 7 [BROADCAST_EDGE] llap - BROADCAST [RS_32] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_87] (rows=250 width=10) - predicate:((value = 'd1value') and key is not null) - TableScan [TS_12] (rows=500 width=10) - default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_92] (rows=1100 width=10) - Conds:SEL_8._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 6 [BROADCAST_EDGE] llap - BROADCAST [RS_29] - PartitionCols:_col3 - Select Operator [SEL_11] (rows=42 width=34) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_86] (rows=42 width=34) - predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null) - TableScan [TS_9] (rows=85 width=34) - default@ss,ss,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"] - <-Select Operator [SEL_8] (rows=1000 width=10) - Output:["_col1"] - Filter Operator [FIL_85] (rows=1000 width=10) - predicate:((key = 'srcpartkey') and value is not null) - TableScan [TS_6] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, z.value, y.value @@ -1156,135 +476,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 10 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Map 9 <- Union 10 (CONTAINS) -Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 13 (BROADCAST_EDGE), Vertex 14 (BROADCAST_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) +Vertex 9 <- Union 10 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_59] - Group By Operator [GBY_57] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_51] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_85] (rows=550 width=10) - Conds:MAPJOIN_84._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 14 [BROADCAST_EDGE] llap - BROADCAST [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_81] (rows=500 width=10) - predicate:key is not null - TableScan [TS_42] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map Join Operator [MAPJOIN_84] (rows=288 width=10) - Conds:SEL_38._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 13 [BROADCAST_EDGE] llap - BROADCAST [RS_46] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_39] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_38] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 10 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_78] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Reducer 3 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_83] (rows=550 width=10) - Conds:MAPJOIN_82._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 8 [BROADCAST_EDGE] llap - BROADCAST [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_77] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map Join Operator [MAPJOIN_82] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 7 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, y.value @@ -1315,272 +519,29 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 17 <- Union 14 (CONTAINS) -Map 20 <- Union 21 (CONTAINS) -Map 27 <- Union 21 (CONTAINS) -Map 28 <- Union 23 (CONTAINS) -Map 29 <- Union 25 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) -Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) -Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) -Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE), Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 18 (BROADCAST_EDGE), Vertex 19 (BROADCAST_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 17 <- Union 14 (CONTAINS) +Vertex 20 <- Union 21 (CONTAINS) +Vertex 22 <- Union 21 (SORT_PARTITION_EDGE), Union 23 (CONTAINS) +Vertex 24 <- Union 23 (SORT_PARTITION_EDGE), Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 6 (CONTAINS), Vertex 30 (BROADCAST_EDGE), Vertex 31 (BROADCAST_EDGE) +Vertex 27 <- Union 21 (CONTAINS) +Vertex 28 <- Union 23 (CONTAINS) +Vertex 29 <- Union 25 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 26 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_170] (rows=550 width=10) - Conds:MAPJOIN_169._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 31 [BROADCAST_EDGE] llap - BROADCAST [RS_112] - PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=10) - predicate:key is not null - TableScan [TS_105] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_169] (rows=484 width=10) - Conds:SEL_101._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 30 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_102] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_101] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_100] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 29 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=10) - predicate:value is not null - TableScan [TS_92] (rows=500 width=10) - Output:["key","value"] - <-Reducer 24 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_91] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_90] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 23 [SIMPLE_EDGE] - <-Map 28 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_82] (rows=500 width=10) - Output:["key","value"] - <-Reducer 22 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_81] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_80] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 21 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=25 width=7) - predicate:value is not null - TableScan [TS_69] (rows=25 width=7) - Output:["key","value"] - <-Map 27 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_74] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:value is not null - TableScan [TS_72] (rows=500 width=10) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_168] (rows=550 width=10) - Conds:MAPJOIN_167._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 19 [BROADCAST_EDGE] llap - BROADCAST [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_167] (rows=419 width=10) - Conds:SEL_48._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 18 [BROADCAST_EDGE] llap - BROADCAST [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 13 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 16 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 3 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_166] (rows=550 width=10) - Conds:MAPJOIN_165._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_165] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 9 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 7 PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -1726,22 +687,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value @@ -1752,42 +698,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key @@ -1801,22 +718,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value @@ -1827,42 +729,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab2,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: explain select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key @@ -1879,61 +752,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_26] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_39] (rows=558 width=10) - Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_36] (rows=500 width=10) - predicate:key is not null - TableScan [TS_15] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_37] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_34] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_33] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - Output:["key"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_35] (rows=242 width=10) - predicate:key is not null - TableScan [TS_10] (rows=242 width=10) - Output:["key"] + Vertex 4 PREHOOK: query: explain select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value @@ -1950,75 +778,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_32] - Group By Operator [GBY_30] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - Group By Operator [GBY_28] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_51] (rows=587 width=10) - Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_47] (rows=500 width=10) - predicate:key is not null - TableScan [TS_21] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_46] (rows=242 width=10) - predicate:key is not null - TableScan [TS_16] (rows=242 width=10) - Output:["key"] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_50] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_48] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_44] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_45] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, y.value @@ -2046,511 +815,34 @@ SELECT x.key, y.value FROM src1 x JOIN src1 y ON (x.key = y.key) JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value) POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 +Plan optimized by CBO. -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 17 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 +Vertex dependency in root stage +Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 9 (CONTAINS) +Vertex 13 <- Union 9 (CONTAINS) +Vertex 16 <- Vertex 17 (BROADCAST_EDGE) +Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 8 <- Union 9 (CONTAINS) - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 18 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 20 [CONTAINS] + <-Vertex 21 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: explain SELECT x.key, y.value @@ -2581,285 +873,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=530 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=484 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_167] (rows=484 width=10) - Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap - BROADCAST [RS_111] - PartitionCols:_col1 - Map Join Operator [MAPJOIN_166] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col0 - Select Operator [SEL_74] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=7) - predicate:key is not null - TableScan [TS_72] (rows=25 width=7) - default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_69] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_107] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_106] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_97] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_96] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_87] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_86] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_77] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=25 width=7) - predicate:value is not null - TableScan [TS_75] (rows=25 width=7) - Output:["key","value"] - <-Map 32 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_80] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=500 width=10) - predicate:value is not null - TableScan [TS_78] (rows=500 width=10) - Output:["key","value"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=577 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=605 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_165] (rows=605 width=10) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_164] (rows=550 width=10) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 19 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 4 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_163] (rows=550 width=10) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_162] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_147] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 8 PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -2923,655 +963,50 @@ INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-4 - Stage-6 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-4 - Stage-7 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-3 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 17 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 - - Stage: Stage-4 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - - Stage: Stage-6 - Stats-Aggr Operator +Plan optimized by CBO. - Stage: Stage-2 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c +Vertex dependency in root stage +Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 9 (CONTAINS) +Vertex 13 <- Union 9 (CONTAINS) +Vertex 16 <- Vertex 17 (BROADCAST_EDGE) +Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 8 <- Union 9 (CONTAINS) - Stage: Stage-7 - Stats-Aggr Operator +Stage-5 + Stats-Aggr Operator + Stage-0 + Move Operator + table:{"name:":"default.a"} + Stage-4 + Dependency Collection{} + Stage-3 + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 18 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 20 [CONTAINS] + <-Vertex 21 [CONTAINS] + <-Vertex 3 [CONTAINS] +Stage-6 + Stats-Aggr Operator + Stage-1 + Move Operator + table:{"name:":"default.b"} + Please refer to the previous Stage-4 +Stage-7 + Stats-Aggr Operator + Stage-2 + Move Operator + table:{"name:":"default.c"} + Please refer to the previous Stage-4 PREHOOK: query: explain FROM @@ -3608,27 +1043,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-5 Stats-Aggr Operator @@ -3638,266 +1073,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 8 llap - File Output Operator [FS_123] - table:{"name:":"default.a"} - Group By Operator [GBY_120] (rows=530 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=484 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_172] (rows=484 width=10) - Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap - BROADCAST [RS_111] - PartitionCols:_col1 - Map Join Operator [MAPJOIN_171] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col0 - Select Operator [SEL_74] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=25 width=7) - predicate:key is not null - TableScan [TS_72] (rows=25 width=7) - default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_69] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_107] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_106] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_166] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_97] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_96] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_165] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_87] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_86] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_77] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=7) - predicate:value is not null - TableScan [TS_75] (rows=25 width=7) - Output:["key","value"] - <-Map 32 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_80] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=10) - predicate:value is not null - TableScan [TS_78] (rows=500 width=10) - Output:["key","value"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=577 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=605 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_170] (rows=605 width=10) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_169] (rows=550 width=10) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 19 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 4 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_168] (rows=550 width=10) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_167] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] - File Output Operator [FS_125] - table:{"name:":"default.b"} - Please refer to the previous Group By Operator [GBY_120] - File Output Operator [FS_127] - table:{"name:":"default.c"} - Please refer to the previous Group By Operator [GBY_120] + Vertex 8 Stage-6 Stats-Aggr Operator Stage-1 @@ -3944,10 +1120,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -3957,51 +1133,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 5 llap - File Output Operator [FS_21] - table:{"name:":"default.dest1"} - Group By Operator [GBY_19] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Group By Operator [GBY_14] (rows=1 width=272) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=1 width=272) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=1 width=272) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - File Output Operator [FS_27] - table:{"name:":"default.dest2"} - Select Operator [SEL_26] (rows=1 width=456) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_25] (rows=1 width=464) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 - Please refer to the previous Group By Operator [GBY_14] + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -4016,33 +1148,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=4400 width=10) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_9] (rows=4400 width=10) - Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=500 width=10) - default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key - TableScan [TS_1] (rows=25 width=7) - default@src1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=2000 width=10) - default@srcpart,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: EXPLAIN SELECT @@ -4059,32 +1171,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=550 width=10) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=550 width=10) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500 width=10) - predicate:key is not null - TableScan [TS_0] (rows=500 width=10) - default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500 width=10) - predicate:key is not null - TableScan [TS_1] (rows=500 width=10) - default@src,b,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: explain FROM ( @@ -4115,11 +1208,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Map 7 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) +Vertex 7 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4129,67 +1222,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap - File Output Operator [FS_19] - table:{"name:":"default.dest1"} - Group By Operator [GBY_17] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Map 7 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_12] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_11] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_12] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap - File Output Operator [FS_25] - table:{"name:":"default.dest2"} - Group By Operator [GBY_23] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + Vertex 4 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -4220,10 +1254,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4233,53 +1267,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap - File Output Operator [FS_16] - table:{"name:":"default.dest1"} - Group By Operator [GBY_14] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_19] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_19] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap - File Output Operator [FS_22] - table:{"name:":"default.dest2"} - Group By Operator [GBY_20] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + Vertex 4 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/explainuser_4.q.out b/ql/src/test/results/clientpositive/llap/explainuser_4.q.out index 4084206..43ff39c 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_4.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_4.q.out @@ -19,39 +19,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -98,41 +73,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=2166 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=2) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=2) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=2) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 3 PREHOOK: query: select count(*) @@ -172,47 +120,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=615 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_21] (rows=2166 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -260,39 +176,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -339,41 +230,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_19] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 3 PREHOOK: query: select count(*) @@ -413,47 +277,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=750 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=750 width=215) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=1501 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Map Join Operator [MAPJOIN_21] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -480,7 +312,7 @@ POSTHOOK: Input: default@alltypesorc -3799 1 10782 1 NULL 6 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Left outer join with residual explain select * @@ -496,26 +328,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Merge Join Operator [MERGEJOIN_9] (rows=150994944 width=431) - Conds:(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],residual filter predicates:{((_col2 = _col14) or _col1 BETWEEN 1 AND 10)} - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - Select Operator [SEL_1] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Select Operator [SEL_3] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_2] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out index 284ffb9..3e8412e 100644 --- a/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out +++ b/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: f @@ -119,7 +119,68 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [f] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col3 + Position of Big Table: 0 + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col0 (type: int) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col5 + Position of Big Table: 0 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: m @@ -194,7 +255,7 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [m] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: g @@ -268,67 +329,6 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [g] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col3 - Position of Big Table: 0 - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col0 (type: int) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col5 - Position of Big Table: 0 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/groupby1.q.out b/ql/src/test/results/clientpositive/llap/groupby1.q.out index ba0a09a..8e2e448 100644 --- a/ql/src/test/results/clientpositive/llap/groupby1.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby1.q.out @@ -27,11 +27,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -48,7 +48,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -63,7 +63,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/groupby2.q.out b/ql/src/test/results/clientpositive/llap/groupby2.q.out index 8286589..59fc6bd 100644 --- a/ql/src/test/results/clientpositive/llap/groupby2.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby2.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,7 +45,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -60,7 +60,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/groupby3.q.out b/ql/src/test/results/clientpositive/llap/groupby3.q.out index be98fd6..9abcac6 100644 --- a/ql/src/test/results/clientpositive/llap/groupby3.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -43,11 +43,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -63,7 +63,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -75,7 +75,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double), _col1 (type: struct), _col2 (type: struct), _col3 (type: string), _col4 (type: string), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/groupby_resolution.q.out b/ql/src/test/results/clientpositive/llap/groupby_resolution.q.out index 53e52ee..31eeafb 100644 --- a/ql/src/test/results/clientpositive/llap/groupby_resolution.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby_resolution.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -30,7 +30,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -66,10 +66,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -85,7 +85,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,11 +121,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -141,7 +141,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -156,7 +156,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -192,11 +192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -212,7 +212,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -227,7 +227,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -263,10 +263,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -289,7 +289,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,10 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -351,7 +351,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -414,7 +414,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -429,7 +429,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -465,11 +465,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -492,7 +492,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -507,7 +507,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -644,12 +644,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -671,7 +671,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -686,7 +686,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -701,7 +701,7 @@ STAGE PLANS: Map-reduce partition columns: 0 (type: int) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -815,7 +815,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/having.q.out b/ql/src/test/results/clientpositive/llap/having.q.out index 9438910..30c7f61 100644 --- a/ql/src/test/results/clientpositive/llap/having.q.out +++ b/ql/src/test/results/clientpositive/llap/having.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -39,7 +39,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -104,10 +104,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -129,7 +129,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -507,7 +507,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -757,10 +757,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -782,7 +782,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -958,10 +958,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -984,7 +984,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1230,10 +1230,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1256,7 +1256,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 8ec11eb..134d588 100644 --- a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -48,11 +48,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -71,7 +71,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -84,7 +84,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -103,21 +118,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -173,11 +173,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -196,7 +196,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -209,7 +209,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -228,21 +243,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -294,11 +294,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -317,7 +317,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -330,7 +330,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -349,21 +364,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -415,11 +415,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -438,7 +438,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -451,7 +451,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -470,21 +485,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -534,11 +534,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -554,7 +554,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -567,7 +567,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -583,21 +598,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -647,11 +647,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -667,7 +667,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -680,7 +680,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -696,21 +711,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -794,11 +794,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -817,7 +817,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -830,7 +830,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -849,21 +864,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -917,11 +917,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -940,7 +940,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -954,7 +954,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -973,21 +988,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1041,11 +1041,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1064,7 +1064,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1077,7 +1077,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1096,21 +1111,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1164,11 +1164,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1187,7 +1187,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1201,7 +1201,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1220,21 +1235,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1297,7 +1297,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1317,10 +1317,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1340,7 +1340,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1355,7 +1355,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1380,7 +1380,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1495,7 +1495,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1515,10 +1515,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1538,7 +1538,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1553,7 +1553,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1578,7 +1578,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out index de81828..48bbe9d 100644 --- a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out +++ b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -60,7 +60,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -77,8 +77,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -91,22 +91,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,6 +106,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -172,11 +172,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -191,7 +191,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -208,8 +208,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -222,22 +222,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -252,6 +237,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -316,11 +316,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -335,7 +335,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -354,9 +354,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -369,7 +369,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -384,7 +399,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -399,21 +414,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -469,11 +469,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -488,7 +488,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -507,9 +507,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -522,7 +522,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -537,7 +552,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -552,21 +567,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -639,14 +639,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -661,7 +663,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -676,7 +678,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -693,8 +695,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -707,7 +709,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -722,7 +757,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -737,7 +772,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -754,8 +789,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -768,40 +803,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -819,8 +821,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -889,14 +889,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -911,7 +913,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -926,7 +928,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -943,8 +945,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -957,7 +959,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -972,7 +1007,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -987,7 +1022,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -1004,8 +1039,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1018,40 +1053,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1069,8 +1071,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1152,11 +1152,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1172,7 +1172,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1190,8 +1190,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1202,8 +1202,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1216,7 +1216,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1231,7 +1246,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1246,7 +1261,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1261,21 +1276,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1347,11 +1347,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1367,7 +1367,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1385,8 +1385,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1397,8 +1397,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1411,7 +1411,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1426,7 +1441,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1441,7 +1456,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1456,21 +1471,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out b/ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out index 3375fa8..af6f03e 100644 --- a/ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out +++ b/ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,23 +45,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '105') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: '105' (type: string) - sort order: + - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -73,7 +57,7 @@ STAGE PLANS: 0 '105' (type: string) 1 '105' (type: string) input vertices: - 1 Reducer 4 + 1 Vertex 4 Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '105' (type: string), 'val_105' (type: string) @@ -86,7 +70,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '105') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: '105' (type: string) + sort order: + + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/insert1.q.out b/ql/src/test/results/clientpositive/llap/insert1.q.out index 4b30156..c156b72 100644 --- a/ql/src/test/results/clientpositive/llap/insert1.q.out +++ b/ql/src/test/results/clientpositive/llap/insert1.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -97,7 +97,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -167,7 +167,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -221,7 +221,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -283,7 +283,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: insert2 diff --git a/ql/src/test/results/clientpositive/llap/insert_into1.q.out b/ql/src/test/results/clientpositive/llap/insert_into1.q.out index c39711b..e72d29d 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -29,7 +29,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -151,7 +151,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -273,7 +273,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -586,7 +586,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/llap/intersect_all.q.out b/ql/src/test/results/clientpositive/llap/intersect_all.q.out index 4dac8a2..fa18977 100644 --- a/ql/src/test/results/clientpositive/llap/intersect_all.q.out +++ b/ql/src/test/results/clientpositive/llap/intersect_all.q.out @@ -146,35 +146,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -197,7 +176,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -218,7 +197,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -248,7 +227,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -269,8 +271,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -799,83 +799,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -898,7 +831,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -919,7 +852,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -940,7 +873,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -970,7 +903,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -991,7 +947,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1012,8 +991,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1542,12 +1542,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1568,28 +1570,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1615,7 +1596,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1645,7 +1626,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1671,8 +1673,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out b/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out index 3698f03..6ac445f 100644 --- a/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/intersect_distinct.q.out @@ -144,35 +144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -195,7 +174,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -216,7 +195,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -239,7 +218,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,8 +262,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -599,83 +599,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -698,7 +631,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -719,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -740,7 +673,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -763,7 +696,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -784,7 +740,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,8 +784,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1144,12 +1144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1170,28 +1172,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1217,7 +1198,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1240,7 +1221,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1266,8 +1268,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/intersect_merge.q.out b/ql/src/test/results/clientpositive/llap/intersect_merge.q.out index 6217785..28658c0 100644 --- a/ql/src/test/results/clientpositive/llap/intersect_merge.q.out +++ b/ql/src/test/results/clientpositive/llap/intersect_merge.q.out @@ -43,38 +43,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 11 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -97,30 +76,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: b @@ -143,51 +120,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -208,7 +141,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -229,7 +162,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -252,28 +185,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -281,7 +216,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -294,36 +229,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -PREHOOK: type: QUERY -POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -346,7 +252,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -369,7 +296,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +PREHOOK: type: QUERY +POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -392,30 +348,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -436,7 +369,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,7 +390,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -480,28 +413,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -509,7 +444,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -522,37 +457,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -575,30 +480,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -621,7 +524,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -644,30 +577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -688,7 +598,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -709,7 +642,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -730,7 +663,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -753,28 +686,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -782,7 +717,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -795,37 +730,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -848,30 +753,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -894,7 +797,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -917,30 +850,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -961,72 +871,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col2 = 5) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1034,7 +902,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1047,7 +915,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1068,60 +936,74 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 = 5) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1144,7 +1026,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1167,7 +1070,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1190,30 +1123,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1234,7 +1144,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1255,7 +1188,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1276,7 +1209,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1299,28 +1232,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1328,7 +1263,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1341,35 +1276,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1392,7 +1299,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1415,7 +1343,35 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1438,7 +1394,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1459,7 +1415,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1482,7 +1438,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1503,7 +1482,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1524,8 +1526,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1546,59 +1546,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1621,7 +1577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1642,7 +1598,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1672,7 +1628,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1693,7 +1672,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1714,8 +1716,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1736,60 +1736,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 7 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1812,7 +1770,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1833,7 +1791,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1854,7 +1812,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1884,7 +1842,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1905,7 +1886,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1939,10 +1920,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/join0.q.out b/ql/src/test/results/clientpositive/llap/join0.q.out index 82440e7..89cfeac 100644 --- a/ql/src/test/results/clientpositive/llap/join0.q.out +++ b/ql/src/test/results/clientpositive/llap/join0.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- SORT_QUERY_RESULTS EXPLAIN @@ -28,11 +28,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -50,25 +50,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -83,7 +65,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -97,6 +79,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -104,7 +104,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN FORMATTED SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM @@ -122,7 +122,7 @@ SELECT src1.key as k1, src1.value as v1, SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 diff --git a/ql/src/test/results/clientpositive/llap/join1.q.out b/ql/src/test/results/clientpositive/llap/join1.q.out index cb0a735..757e286 100644 --- a/ql/src/test/results/clientpositive/llap/join1.q.out +++ b/ql/src/test/results/clientpositive/llap/join1.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -51,27 +51,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -94,6 +74,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 46d6281..d593796 100644 --- a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -45,11 +45,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -66,13 +66,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 25 + Estimated key counts: Vertex 3 => 25 keys: 0 _col0 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 58 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -137,7 +137,55 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: x @@ -211,7 +259,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -285,54 +333,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -509,12 +509,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -531,13 +531,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 + Estimated key counts: Vertex 4 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Position of Big Table: 0 Statistics: Num rows: 44 Data size: 11616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -600,7 +600,77 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3 + Position of Big Table: 1 + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: _col3 (type: string) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: z @@ -674,7 +744,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [z] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: w @@ -747,7 +817,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [w] - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y @@ -821,76 +891,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - Position of Big Table: 1 - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: _col3 (type: string) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 0 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -1063,11 +1063,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1142,7 +1142,55 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 0 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1159,13 +1207,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 + Estimated key counts: Vertex 4 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Position of Big Table: 0 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator @@ -1232,7 +1280,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1286,74 +1334,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src1 - numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 - Truncated Path -> Alias: - /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 0 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 + Truncated Path -> Alias: + /src1 [x] Stage: Stage-2 Dependency Collection @@ -1528,11 +1528,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1607,7 +1607,55 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 0 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1676,7 +1724,33 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Vertex 5 Map Operator Tree: TableScan alias: x @@ -1750,80 +1824,6 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 0 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true Stage: Stage-2 Dependency Collection @@ -2010,11 +2010,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2033,7 +2033,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2053,7 +2076,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2067,7 +2090,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2087,29 +2110,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection @@ -2266,11 +2266,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -2289,7 +2289,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2309,7 +2332,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2323,7 +2346,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2343,29 +2366,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/join46.q.out b/ql/src/test/results/clientpositive/llap/join46.q.out index 1cd856b..fc28c6d 100644 --- a/ql/src/test/results/clientpositive/llap/join46.q.out +++ b/ql/src/test/results/clientpositive/llap/join46.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -77,24 +77,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -112,6 +95,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -166,10 +166,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -186,27 +186,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -227,6 +207,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -258,7 +258,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Conjunction with pred on single inputs (left outer join) EXPLAIN SELECT * @@ -282,10 +282,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -300,25 +300,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -339,6 +321,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -346,7 +346,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -390,10 +390,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -410,24 +410,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -445,6 +428,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -472,7 +472,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Condition on one input (left outer join) EXPLAIN SELECT * @@ -494,10 +494,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -512,22 +512,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -548,6 +533,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -555,7 +555,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -582,7 +582,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (left outer join) EXPLAIN SELECT * @@ -608,10 +608,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -626,22 +626,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -660,6 +645,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -667,7 +667,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -699,7 +699,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (left outer join) EXPLAIN SELECT * @@ -723,10 +723,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -741,22 +741,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -775,6 +760,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -782,7 +782,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -812,7 +812,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (left outer join) EXPLAIN SELECT * @@ -836,10 +836,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -854,22 +854,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -888,6 +873,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -895,7 +895,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -945,10 +945,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -965,24 +965,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1001,6 +984,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1033,7 +1033,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat 102 2 Del 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Complex condition, projecting columns EXPLAIN SELECT col_1, col_2 @@ -1057,10 +1057,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1075,22 +1075,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1113,6 +1098,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1120,7 +1120,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT col_1, col_2 FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -1145,7 +1145,7 @@ Mat Ema Bob NULL Car Del Car Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (right outer join) EXPLAIN SELECT * @@ -1171,10 +1171,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1189,22 +1189,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1223,6 +1208,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1230,7 +1230,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1262,7 +1262,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (right outer join) EXPLAIN SELECT * @@ -1286,10 +1286,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1304,22 +1304,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1338,6 +1323,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1345,7 +1345,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1372,7 +1372,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (right outer join) EXPLAIN SELECT * @@ -1396,10 +1396,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1414,22 +1414,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1448,6 +1433,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1455,7 +1455,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1507,10 +1507,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1527,24 +1527,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1563,6 +1546,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1593,7 +1593,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (full outer join) EXPLAIN SELECT * @@ -1619,10 +1619,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1637,22 +1637,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1671,6 +1656,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1678,7 +1678,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1710,7 +1710,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (full outer join) EXPLAIN SELECT * @@ -1734,10 +1734,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1752,22 +1752,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1786,6 +1771,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1793,7 +1793,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1823,7 +1823,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (full outer join) EXPLAIN SELECT * @@ -1847,10 +1847,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1865,22 +1865,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1899,6 +1884,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1906,7 +1906,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1958,10 +1958,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1978,24 +1978,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2014,6 +1997,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2048,7 +2048,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- Mixed ( FOJ (ROJ, LOJ) ) EXPLAIN SELECT * @@ -2100,46 +2100,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -2156,24 +2122,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2189,7 +2138,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2208,7 +2157,41 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2224,6 +2207,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2231,7 +2231,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out b/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out index f576f81..4761d45 100644 --- a/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out +++ b/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -77,24 +77,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -115,6 +98,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -143,7 +143,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Condition on one input (left outer join) EXPLAIN SELECT * @@ -165,10 +165,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -183,22 +183,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -219,6 +204,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -226,7 +226,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git a/ql/src/test/results/clientpositive/llap/join_filters.q.out b/ql/src/test/results/clientpositive/llap/join_filters.q.out index 6eebf83..321cdc3 100644 --- a/ql/src/test/results/clientpositive/llap/join_filters.q.out +++ b/ql/src/test/results/clientpositive/llap/join_filters.q.out @@ -18,7 +18,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -29,7 +29,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -43,7 +43,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -57,7 +57,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -360,7 +360,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -411,7 +411,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -774,7 +774,7 @@ NULL NULL 48 NULL NULL NULL NULL 135 NULL NULL NULL 35 UBr9lyqgsjDFvooMgQlZ9w== -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -785,7 +785,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -799,7 +799,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -813,7 +813,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1116,7 +1116,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1167,7 +1167,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/join_nulls.q.out b/ql/src/test/results/clientpositive/llap/join_nulls.q.out index a1411cd..3756290 100644 --- a/ql/src/test/results/clientpositive/llap/join_nulls.q.out +++ b/ql/src/test/results/clientpositive/llap/join_nulls.q.out @@ -18,7 +18,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -54,7 +54,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -281,7 +281,7 @@ POSTHOOK: Input: default@myinput1 100 100 100 100 100 100 NULL 35 NULL 35 NULL 35 NULL NULL NULL NULL 48 NULL -Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/join_nullsafe.q.out b/ql/src/test/results/clientpositive/llap/join_nullsafe.q.out index 7e3b67d..8071328 100644 --- a/ql/src/test/results/clientpositive/llap/join_nullsafe.q.out +++ b/ql/src/test/results/clientpositive/llap/join_nullsafe.q.out @@ -33,10 +33,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -49,20 +49,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,6 +72,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -145,7 +145,31 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int) + 1 value (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -161,7 +185,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -177,30 +201,6 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int) - 1 value (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -231,10 +231,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -247,33 +247,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -298,6 +272,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -355,10 +355,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -373,7 +373,32 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int), value (type: int) + 1 value (type: int), key (type: int) + 2 key (type: int), value (type: int) + nullSafes: [true, false] + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -388,7 +413,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -403,31 +428,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int), value (type: int) - 1 value (type: int), key (type: int) - 2 key (type: int), value (type: int) - nullSafes: [true, false] - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -473,31 +473,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int), key (type: int) - sort order: ++ - Map-reduce partition columns: value (type: int), key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: int) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -522,6 +498,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int), key (type: int) + sort order: ++ + Map-reduce partition columns: value (type: int), key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: int) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1605,10 +1605,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1624,23 +1624,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is null (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: null (type: int) - sort order: + - Map-reduce partition columns: null (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1663,6 +1647,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: null (type: int) + sort order: + + Map-reduce partition columns: null (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/lateral_view.q.out b/ql/src/test/results/clientpositive/llap/lateral_view.q.out index bb3cfcf..7856fd9 100644 --- a/ql/src/test/results/clientpositive/llap/lateral_view.q.out +++ b/ql/src/test/results/clientpositive/llap/lateral_view.q.out @@ -28,11 +28,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -78,7 +78,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -94,7 +94,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -630,10 +630,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_pyang_src_rcfile @@ -654,7 +654,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index 81e9ade..49ab5f3 100644 --- a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -39,24 +39,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -77,6 +60,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -120,10 +120,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -140,24 +140,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -178,6 +161,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -221,11 +221,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -244,24 +244,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -277,7 +260,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -298,6 +281,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -347,12 +347,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -371,41 +371,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,7 +387,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -438,7 +404,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -463,6 +429,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -518,13 +518,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -541,43 +541,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -598,7 +562,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -614,7 +597,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -633,7 +616,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,6 +632,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -706,14 +706,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -730,40 +730,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -780,7 +747,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,7 +764,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -818,7 +801,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -834,7 +817,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -949,10 +949,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -969,24 +969,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1008,6 +991,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1043,38 +1043,21 @@ on src1.key = src2.key limit 1 offset 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: src2 + alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -1088,7 +1071,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1110,6 +1093,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1153,11 +1153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1177,24 +1177,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1211,7 +1194,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1233,6 +1216,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1281,12 +1281,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -1306,41 +1306,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1357,7 +1323,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1374,7 +1340,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1400,6 +1366,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1454,13 +1454,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1477,44 +1477,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Offset of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1536,7 +1499,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Offset of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1553,7 +1536,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1573,7 +1556,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1590,6 +1573,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1646,14 +1646,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1670,40 +1670,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1720,7 +1687,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1738,7 +1705,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1759,7 +1742,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1776,7 +1759,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index 3fe4837..3d8bcf3 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -21,10 +21,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -108,10 +108,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -128,7 +128,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -195,10 +195,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -222,7 +222,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -293,10 +293,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -320,7 +320,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -391,10 +391,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -416,7 +416,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -508,7 +508,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -582,10 +582,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -606,7 +606,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -682,10 +682,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -708,7 +708,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,11 +805,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -832,7 +832,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -847,7 +847,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -924,13 +924,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -954,31 +954,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.3 - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -999,7 +975,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1020,7 +996,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1037,7 +1037,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1080,10 +1080,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1101,7 +1101,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1172,10 +1172,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1192,7 +1192,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1341,11 +1341,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1362,7 +1362,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1380,7 +1380,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index 229f35a..f75ab91 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -120,7 +120,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -187,11 +187,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -215,7 +215,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -230,7 +230,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -299,11 +299,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -327,7 +327,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -342,7 +342,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -411,11 +411,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -437,7 +437,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -450,7 +450,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4265 Data size: 25480 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -517,11 +517,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -542,7 +542,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -562,7 +562,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -629,11 +629,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -654,7 +654,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -674,7 +674,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -743,11 +743,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -770,7 +770,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -785,7 +785,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -880,11 +880,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -907,7 +907,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -922,7 +922,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -991,11 +991,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1013,7 +1013,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1028,7 +1028,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1097,10 +1097,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1117,7 +1117,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1266,11 +1266,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1287,7 +1287,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1305,7 +1305,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 52ae100..1fbaae5 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -180,7 +180,7 @@ PREHOOK: Input: default@src1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY @@ -370,7 +370,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_dp1 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: from src_dp, src_dp1 insert into dest_dp1 partition (year) select first, word, year insert into dest_dp2 partition (y, m) select first, word, year, month diff --git a/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out b/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index 2af0d6e..d41f9e0 100644 --- a/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ b/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -63,7 +63,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out index c755268..ded4a76 100644 --- a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out @@ -39,7 +39,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -155,11 +155,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -178,25 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src_orc - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -214,7 +196,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -259,11 +259,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -282,7 +284,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src_orc @@ -302,22 +319,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -332,8 +334,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -366,10 +366,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -385,23 +385,7 @@ STAGE PLANS: value expressions: key (type: string), ds (type: string), hr (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), ds (type: string), hr (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -423,6 +407,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), ds (type: string), hr (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index 541ece8..43c6d12 100644 --- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1623,10 +1623,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: oft @@ -1639,7 +1639,7 @@ STAGE PLANS: 1 ctinyint (type: tinyint) outputColumnNames: _col1, _col6, _col7, _col10 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 960 Data size: 497280 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255)) @@ -1654,7 +1654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: od @@ -1678,7 +1678,7 @@ STAGE PLANS: Target Input: oft Partition key expr: ctinyint Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: vectorized, llap LLAP IO: all inputs diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out index 53801d8..8046ad8 100644 --- a/ql/src/test/results/clientpositive/llap/llap_udf.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -99,7 +99,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -141,7 +141,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -189,7 +189,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -239,7 +239,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -277,7 +277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc diff --git a/ql/src/test/results/clientpositive/llap/llapdecider.q.out b/ql/src/test/results/clientpositive/llap/llapdecider.q.out index c7339c9..6a449ae 100644 --- a/ql/src/test/results/clientpositive/llap/llapdecider.q.out +++ b/ql/src/test/results/clientpositive/llap/llapdecider.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -38,7 +38,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -51,7 +51,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -96,11 +96,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -121,7 +121,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -134,7 +134,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -167,11 +167,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -194,7 +194,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -208,7 +208,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -250,11 +250,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -277,7 +277,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -291,7 +291,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -325,11 +325,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -349,25 +349,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -382,7 +364,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -395,6 +377,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -415,11 +415,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -439,27 +439,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -475,7 +455,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -489,6 +469,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -509,11 +509,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -533,25 +533,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -567,7 +549,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -581,6 +563,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -601,11 +601,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -623,25 +623,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -656,7 +638,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -669,6 +651,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -689,11 +689,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -711,25 +711,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -744,7 +726,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -757,6 +739,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -777,11 +777,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -801,27 +801,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -836,7 +816,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -849,6 +829,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -869,11 +869,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -891,25 +891,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -924,7 +906,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -937,6 +919,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -957,11 +957,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -981,27 +981,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1017,7 +997,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1031,6 +1011,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1057,10 +1057,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1083,7 +1083,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1118,10 +1118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1142,7 +1142,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1176,10 +1176,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1200,7 +1200,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1234,10 +1234,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1260,7 +1260,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out index efd79a6..fb516b6 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out @@ -69,7 +69,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out index 52d754a..f71ea87 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out @@ -49,10 +49,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -68,7 +68,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out index 7a5b0fc..520e97d 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out @@ -55,7 +55,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index 9c4e8a8..13af9d1 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -38,7 +38,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git a/ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out index 366eda0..0fb51b2 100644 --- a/ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out @@ -111,10 +111,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: expod1 @@ -143,7 +143,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -181,7 +181,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -199,7 +199,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: expod2 diff --git a/ql/src/test/results/clientpositive/llap/mapjoin2.q.out b/ql/src/test/results/clientpositive/llap/mapjoin2.q.out index cdee70b..3a74b84 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin2.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin2.q.out @@ -22,7 +22,7 @@ POSTHOOK: type: QUERY POSTHOOK: Output: default@tbl POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -32,7 +32,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### 1 one true true -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -42,7 +42,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### true true 2 two -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl diff --git a/ql/src/test/results/clientpositive/llap/mapjoin3.q.out b/ql/src/test/results/clientpositive/llap/mapjoin3.q.out index cf1bebb..1aa212e 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin3.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin3.q.out @@ -98,10 +98,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -118,7 +118,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col2 (type: varchar(100)), _col3 (type: varchar(100)) @@ -133,7 +133,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_1 diff --git a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out index cf8912d..9b430df 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -77,7 +77,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -88,7 +88,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -159,10 +159,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -182,7 +182,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -193,7 +193,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -244,7 +244,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Conjunction with pred on single inputs (left outer join) EXPLAIN SELECT * @@ -268,10 +268,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -291,7 +291,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 116 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -302,7 +302,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -327,7 +327,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -371,10 +371,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -391,7 +391,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -408,7 +408,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -446,7 +446,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Condition on one input (left outer join) EXPLAIN SELECT * @@ -468,10 +468,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -491,7 +491,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -502,7 +502,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -524,7 +524,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -551,7 +551,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (left outer join) EXPLAIN SELECT * @@ -577,10 +577,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -597,7 +597,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -609,7 +609,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -631,7 +631,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -663,7 +663,7 @@ NULL NULL None 102 2 Del 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (left outer join) EXPLAIN SELECT * @@ -687,10 +687,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -707,7 +707,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -719,7 +719,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -741,7 +741,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -771,7 +771,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (left outer join) EXPLAIN SELECT * @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -815,7 +815,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -827,7 +827,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -849,7 +849,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -899,10 +899,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -919,7 +919,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -931,7 +931,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -980,7 +980,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (right outer join) EXPLAIN SELECT * @@ -1006,10 +1006,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1024,7 +1024,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1041,7 +1041,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1060,7 +1060,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1092,7 +1092,7 @@ NULL NULL None 102 2 Del 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (right outer join) EXPLAIN SELECT * @@ -1116,10 +1116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1134,7 +1134,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1151,7 +1151,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1170,7 +1170,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1197,7 +1197,7 @@ POSTHOOK: Input: default@test2 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (right outer join) EXPLAIN SELECT * @@ -1221,10 +1221,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1239,7 +1239,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1256,7 +1256,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1275,7 +1275,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1327,10 +1327,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1347,7 +1347,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1364,7 +1364,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1406,7 +1406,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and single inputs (full outer join) EXPLAIN SELECT * @@ -1432,10 +1432,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1450,22 +1450,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1484,6 +1469,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1491,7 +1491,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1523,7 +1523,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and left input (full outer join) EXPLAIN SELECT * @@ -1547,10 +1547,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1565,22 +1565,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1599,6 +1584,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1606,7 +1606,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1636,7 +1636,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- Disjunction with pred on multiple inputs and right input (full outer join) EXPLAIN SELECT * @@ -1660,10 +1660,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1678,22 +1678,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1712,6 +1697,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1719,7 +1719,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1771,10 +1771,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1791,24 +1791,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1827,6 +1810,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1861,7 +1861,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- Mixed ( FOJ (ROJ, LOJ) ) EXPLAIN SELECT * @@ -1913,12 +1913,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 4 <- Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1935,7 +1935,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1952,7 +1952,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1961,7 +1961,26 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -1978,7 +1997,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 5 + 1 Vertex 5 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1987,7 +2006,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: test2 @@ -2004,25 +2023,6 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2030,7 +2030,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out index 01f1f80..99a4022 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out @@ -91,11 +91,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -115,7 +115,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(4,2)) @@ -124,7 +124,21 @@ STAGE PLANS: value expressions: _col1 (type: decimal(4,0)) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -143,20 +157,6 @@ STAGE PLANS: Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out index f5f1bad..50dcf32 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -80,7 +80,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -91,7 +91,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -136,7 +136,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Condition on one input (left outer join) EXPLAIN SELECT * @@ -158,10 +158,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -181,7 +181,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -192,7 +192,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -214,7 +214,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out index e129556..b8b0dd4 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -39,25 +39,25 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 2 => 25 + Estimated key counts: Vertex 2 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Position of Big Table: 0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 500 + Estimated key counts: Vertex 3 => 500 keys: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -276,7 +276,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -349,7 +349,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [src1] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -444,10 +444,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -467,7 +467,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 732 Data size: 7782 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -477,7 +477,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 805 Data size: 8560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -488,7 +488,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -507,7 +507,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -548,11 +548,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -572,7 +572,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -582,7 +582,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -598,7 +598,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src @@ -617,7 +637,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src1 @@ -636,26 +656,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/merge1.q.out b/ql/src/test/results/clientpositive/llap/merge1.q.out index 4ec5d06..69c909b 100644 --- a/ql/src/test/results/clientpositive/llap/merge1.q.out +++ b/ql/src/test/results/clientpositive/llap/merge1.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -55,7 +55,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -483,7 +483,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -549,7 +549,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git a/ql/src/test/results/clientpositive/llap/merge2.q.out b/ql/src/test/results/clientpositive/llap/merge2.q.out index 383c598..e10835f 100644 --- a/ql/src/test/results/clientpositive/llap/merge2.q.out +++ b/ql/src/test/results/clientpositive/llap/merge2.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -55,7 +55,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -483,7 +483,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -549,7 +549,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out b/ql/src/test/results/clientpositive/llap/mergejoin.q.out index b73b427..0bf00b0 100644 --- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out @@ -17,10 +17,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -41,7 +41,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -62,24 +80,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -261,11 +261,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -285,27 +285,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -324,7 +304,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -339,6 +319,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1351,11 +1351,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1371,23 +1371,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1406,7 +1390,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1421,6 +1405,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1462,11 +1462,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1482,23 +1482,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1517,7 +1501,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1532,6 +1516,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1573,11 +1573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1593,23 +1593,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1628,7 +1612,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1643,6 +1627,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1680,12 +1680,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1706,47 +1706,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1762,7 +1722,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1781,7 +1741,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1796,6 +1756,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1833,11 +1833,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1857,27 +1857,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1896,7 +1876,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1911,6 +1891,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1954,13 +1954,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 8 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1980,7 +1982,57 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 266 Data size: 27152 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 558 Data size: 57019 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s3 @@ -2000,7 +2052,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s2 @@ -2020,7 +2072,7 @@ STAGE PLANS: Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: b @@ -2040,58 +2092,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 266 Data size: 27152 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 558 Data size: 57019 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 5 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -2112,11 +2112,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2136,27 +2136,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2175,7 +2155,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2190,6 +2170,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2225,12 +2225,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2251,47 +2251,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2307,7 +2267,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2326,7 +2286,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2341,6 +2301,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2386,13 +2386,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 8 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2412,7 +2414,57 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 266 Data size: 27152 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 558 Data size: 57019 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s3 @@ -2432,7 +2484,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s2 @@ -2452,7 +2504,7 @@ STAGE PLANS: Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: b @@ -2472,58 +2524,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 266 Data size: 27152 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 558 Data size: 57019 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 5 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -2558,13 +2558,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -2583,26 +2583,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: t2 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2614,7 +2595,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2633,7 +2614,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2648,7 +2629,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: t2 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2694,7 +2694,7 @@ POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### 480 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2714,7 +2714,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2729,7 +2729,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2750,8 +2750,8 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2774,8 +2774,8 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -3302,8 +3302,8 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -3824,7 +3824,7 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[25][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -3845,7 +3845,7 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join diff --git a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out index 04b556e..801533e 100644 --- a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out +++ b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out @@ -191,10 +191,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -251,10 +251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -274,7 +274,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -311,10 +311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -334,7 +334,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -625,10 +625,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -648,7 +648,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out index 1eafc46..cc3053a 100644 --- a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -46,7 +46,7 @@ STAGE PLANS: auto parallelism: false Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -113,10 +113,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -189,7 +189,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -251,10 +251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -328,7 +328,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -390,10 +390,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -467,7 +467,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: /test1/ds=1 [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -538,12 +538,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a2 @@ -659,7 +659,62 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [a2] /test1/ds=2 [a2] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -779,62 +834,7 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [test1] /test1/ds=2 [test1] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -869,7 +869,7 @@ STAGE PLANS: Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Stage: Stage-0 Fetch Operator @@ -932,10 +932,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1105,7 +1105,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1171,10 +1171,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1346,7 +1346,7 @@ STAGE PLANS: /test2/ds=1/hr=1 [test2] /test2/ds=1/hr=2 [test2] /test2/ds=1/hr=3 [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1412,10 +1412,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1534,7 +1534,7 @@ STAGE PLANS: Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] nullscan://null/default.test1/part_ds=2_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1649,10 +1649,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1916,7 +1916,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/mrr.q.out b/ql/src/test/results/clientpositive/llap/mrr.q.out index 6272d04..1b4241e 100644 --- a/ql/src/test/results/clientpositive/llap/mrr.q.out +++ b/ql/src/test/results/clientpositive/llap/mrr.q.out @@ -17,11 +17,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -58,7 +58,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -411,12 +411,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -435,27 +435,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -476,7 +456,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -494,7 +474,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -508,6 +488,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -847,12 +847,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -871,7 +871,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: s2 @@ -891,7 +891,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col1 (type: string), _col2 (type: string) @@ -905,7 +905,7 @@ STAGE PLANS: Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -923,7 +923,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1304,61 +1304,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1380,7 +1336,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1394,7 +1350,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1407,7 +1363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1421,7 +1377,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1434,7 +1390,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1456,7 +1412,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 19380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1470,7 +1426,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1487,7 +1465,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1500,6 +1478,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1700,11 +1700,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1726,27 +1726,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1763,14 +1743,14 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1784,6 +1764,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out index eac269c..0409bb2 100644 --- a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out +++ b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out @@ -184,11 +184,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -208,7 +208,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -221,7 +221,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -234,7 +234,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -253,7 +268,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -272,21 +287,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -351,11 +351,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -375,7 +375,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -388,7 +388,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -401,7 +401,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -420,7 +435,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -439,21 +454,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -522,11 +522,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -546,7 +546,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -560,7 +560,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -576,7 +576,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -595,7 +615,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -615,26 +635,6 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -829,11 +829,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -853,7 +853,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -867,7 +867,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -881,7 +881,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -895,7 +895,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -912,7 +912,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -931,7 +946,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -951,7 +966,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -970,7 +985,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -989,21 +1004,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1170,11 +1170,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1194,7 +1194,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1208,7 +1208,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1222,7 +1222,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1236,7 +1236,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1253,7 +1253,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1272,7 +1287,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1292,7 +1307,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1311,7 +1326,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1330,21 +1345,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1513,11 +1513,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1537,7 +1537,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1551,7 +1551,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1565,7 +1565,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1579,7 +1579,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1596,7 +1596,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1615,7 +1630,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1635,7 +1650,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1654,7 +1669,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1673,21 +1688,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1852,14 +1852,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1879,7 +1879,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: smalltbl4 @@ -1898,65 +1898,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: smalltbl3 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1977,7 +1919,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1998,7 +1940,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2019,7 +1961,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2043,7 +1985,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2058,6 +2000,64 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 7 + Map Operator Tree: + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: smalltbl3 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out b/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out index ae47be1..8f601d5 100644 --- a/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out +++ b/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out @@ -23,12 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -48,7 +50,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -56,7 +58,21 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -75,7 +91,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -95,7 +111,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -103,7 +119,7 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -122,22 +138,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -275,12 +275,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -299,7 +301,36 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y1 @@ -318,7 +349,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x2 @@ -337,7 +368,22 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y2 @@ -356,52 +402,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -533,12 +533,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -559,7 +561,33 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -579,7 +607,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -587,7 +615,7 @@ STAGE PLANS: Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -606,34 +634,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -760,15 +760,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -788,7 +788,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -802,7 +802,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -821,7 +878,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -841,7 +898,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -855,7 +912,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -874,76 +944,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -1036,15 +1036,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1064,7 +1064,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1078,7 +1078,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -1097,7 +1154,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -1117,7 +1174,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1131,7 +1188,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -1150,76 +1220,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -1318,14 +1318,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1345,7 +1345,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1359,46 +1359,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1414,7 +1375,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1428,7 +1389,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1441,7 +1402,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1455,7 +1416,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1560,14 +1560,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1587,7 +1587,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1601,46 +1601,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1656,7 +1617,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1670,7 +1631,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1683,7 +1644,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1697,7 +1658,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1821,11 +1821,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1844,7 +1844,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 137 Data size: 2487 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1857,7 +1857,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1876,21 +1891,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1951,13 +1951,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1973,7 +1975,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1987,7 +1989,36 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col1 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2002,7 +2033,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2017,7 +2048,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: a @@ -2033,7 +2064,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -2047,37 +2078,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2093,7 +2094,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1 input vertices: - 1 Map 5 + 1 Vertex 9 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -2106,37 +2107,36 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 9 + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: c Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/multi_insert.q.out b/ql/src/test/results/clientpositive/llap/multi_insert.q.out index d8052c6..ba09d1d 100644 --- a/ql/src/test/results/clientpositive/llap/multi_insert.q.out +++ b/ql/src/test/results/clientpositive/llap/multi_insert.q.out @@ -41,7 +41,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -184,7 +184,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -327,7 +327,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -470,7 +470,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -613,10 +613,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -631,7 +631,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -767,10 +767,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -785,7 +785,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -921,10 +921,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -939,7 +939,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1075,10 +1075,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1093,7 +1093,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1229,11 +1229,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1266,7 +1268,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1299,8 +1301,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1426,11 +1426,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1463,7 +1465,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1496,8 +1498,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1623,11 +1623,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1660,7 +1662,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1693,8 +1695,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1820,11 +1820,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1857,7 +1859,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1890,8 +1892,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -2015,7 +2015,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2118,7 +2118,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2221,7 +2221,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2324,7 +2324,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git a/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out index eb67a6e..50da4cb 100644 --- a/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out @@ -71,7 +71,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -301,11 +301,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -415,7 +415,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -553,11 +553,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -616,7 +616,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -633,7 +633,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward @@ -824,12 +824,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -949,7 +949,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -966,7 +966,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -983,7 +983,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1182,12 +1182,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -1284,7 +1284,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1301,7 +1301,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1318,7 +1318,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Forward diff --git a/ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out b/ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out index b45e4db..2b663c5 100644 --- a/ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out +++ b/ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out @@ -124,12 +124,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=500 width=184) - Output:["_col0"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 PREHOOK: query: -- Simple Scans without limit (will be Fetch task now) explain select key, value from src diff --git a/ql/src/test/results/clientpositive/llap/offset_limit.q.out b/ql/src/test/results/clientpositive/llap/offset_limit.q.out index adfeb05..b6b29be 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -56,7 +56,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index f56318d..eb55c91 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -101,10 +101,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -121,7 +121,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -315,7 +315,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -387,10 +387,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -412,7 +412,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -505,7 +505,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -580,10 +580,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -604,7 +604,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -681,10 +681,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -707,7 +707,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,11 +805,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -832,7 +832,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -847,7 +847,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -917,10 +917,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -938,7 +938,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1010,10 +1010,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1030,7 +1030,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1150,11 +1150,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1171,7 +1171,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1189,7 +1189,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1317,13 +1317,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1340,24 +1340,7 @@ STAGE PLANS: TopN Hash Memory Usage: 2.0E-5 Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 2.0E-5 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1379,7 +1362,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1401,7 +1384,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 2.0E-5 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1419,7 +1419,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out index d9775b5..254d318 100644 --- a/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out @@ -56,10 +56,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -86,7 +86,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -152,11 +152,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -228,7 +228,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -441,42 +476,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -535,11 +535,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -611,7 +613,38 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: srcpart @@ -827,7 +860,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Reducer 2 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -858,39 +891,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -918,7 +918,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 0 2000 -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain extended select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY @@ -934,11 +934,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1009,7 +1009,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -1222,42 +1257,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1282,7 +1282,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -1316,11 +1316,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1407,7 +1409,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1475,27 +1477,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 5 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1530,8 +1512,26 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -1564,10 +1564,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1636,7 +1636,46 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s1] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1705,45 +1744,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s2] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1774,10 +1774,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1849,7 +1849,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/orc_llap.q.out b/ql/src/test/results/clientpositive/llap/orc_llap.q.out index 74a6b29..50c33c9 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap.q.out @@ -81,7 +81,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: default@cross_numbers POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -121,7 +121,7 @@ POSTHOOK: Output: default@orc_llap_small POSTHOOK: Lineage: orc_llap_small.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: orc_llap_small.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_llap_small.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- Cross join with no projection - do it on small table explain select count(1) from orc_llap_small y join orc_llap_small x @@ -139,11 +139,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -157,7 +157,7 @@ STAGE PLANS: 0 1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 225 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -170,19 +170,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -197,6 +185,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -204,7 +204,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY PREHOOK: Input: default@orc_llap_small @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -280,7 +280,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -326,10 +326,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -353,7 +353,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -399,10 +399,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -426,7 +426,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -472,11 +472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -499,7 +499,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -521,7 +521,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -567,11 +567,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -592,28 +592,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -637,7 +616,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -652,6 +631,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -668,7 +668,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_llap #### A masked pattern was here #### -735462183586256 -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- multi-stripe test insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 @@ -720,10 +720,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -747,7 +747,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -793,10 +793,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -820,7 +820,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -866,10 +866,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -893,7 +893,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -939,11 +939,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -966,7 +966,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -988,7 +988,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1034,11 +1034,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -1059,28 +1059,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1104,7 +1083,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1119,6 +1098,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 81f1d74..e90921b 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -278,9 +278,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -317,9 +317,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -344,9 +344,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -369,9 +369,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -395,9 +395,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -421,9 +421,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -447,9 +447,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -472,9 +472,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -498,9 +498,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -524,9 +524,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -550,9 +550,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -576,9 +576,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -602,9 +602,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -628,9 +628,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -683,9 +683,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -703,9 +703,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -744,9 +744,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -772,9 +772,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -797,9 +797,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -838,9 +838,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -864,9 +864,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -885,9 +885,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -910,9 +910,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -935,9 +935,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -961,9 +961,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -987,9 +987,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1013,9 +1013,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1039,9 +1039,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1065,9 +1065,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1091,9 +1091,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1118,9 +1118,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1138,9 +1138,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1158,9 +1158,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1179,9 +1179,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1205,9 +1205,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1231,9 +1231,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1257,9 +1257,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 885871f..610a4b3 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -278,9 +278,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -305,9 +305,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 01adcd4..d5fca30 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -80,7 +80,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector @@ -231,7 +231,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector diff --git a/ql/src/test/results/clientpositive/llap/orc_merge5.q.out b/ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 18ccba0..cb6491b 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -141,7 +141,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge6.q.out b/ql/src/test/results/clientpositive/llap/orc_merge6.q.out index b38bffc..262518b 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -45,7 +45,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -188,7 +188,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 595061e..f97c80d 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -47,7 +47,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -224,7 +224,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out index 3f8d559..f08fec9 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index e6bfd33..64703a2 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -45,7 +45,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index bf94965..31a031a 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -229,9 +229,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -268,9 +268,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -295,9 +295,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -320,9 +320,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -346,9 +346,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -372,9 +372,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -398,9 +398,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -423,9 +423,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -449,9 +449,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -475,9 +475,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -501,9 +501,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -527,9 +527,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -553,9 +553,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -579,9 +579,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -634,9 +634,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -654,9 +654,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -695,9 +695,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -723,9 +723,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -748,9 +748,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -789,9 +789,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -815,9 +815,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -836,9 +836,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -861,9 +861,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -886,9 +886,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -912,9 +912,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -938,9 +938,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -964,9 +964,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -990,9 +990,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1016,9 +1016,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1042,9 +1042,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1069,9 +1069,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1089,9 +1089,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1109,9 +1109,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1130,9 +1130,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1156,9 +1156,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1182,9 +1182,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1208,9 +1208,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1234,9 +1234,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -1262,9 +1262,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1287,9 +1287,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1312,9 +1312,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index bd81989..b60ce41 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -229,9 +229,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -271,9 +271,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -297,9 +297,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -323,9 +323,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -349,9 +349,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -379,9 +379,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -414,9 +414,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -434,9 +434,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -454,9 +454,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -474,9 +474,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -498,9 +498,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -533,9 +533,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -553,9 +553,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -573,9 +573,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -593,9 +593,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -617,9 +617,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -652,9 +652,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -672,9 +672,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -692,9 +692,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -712,9 +712,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -736,9 +736,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -756,9 +756,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -776,9 +776,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -796,9 +796,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -816,9 +816,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -836,9 +836,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -856,9 +856,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -884,9 +884,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -913,9 +913,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -932,9 +932,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -955,9 +955,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -974,9 +974,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -994,9 +994,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -1022,9 +1022,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1051,9 +1051,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1070,9 +1070,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1093,9 +1093,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1112,9 +1112,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1135,9 +1135,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1154,9 +1154,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1177,9 +1177,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1202,9 +1202,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1232,9 +1232,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1251,9 +1251,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1270,9 +1270,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1289,9 +1289,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 diff --git a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out index 48a86cf..83d14fb 100644 --- a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out @@ -134,10 +134,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -157,7 +157,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -192,10 +192,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -215,7 +215,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -330,10 +330,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -356,7 +356,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -397,10 +397,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -424,7 +424,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -598,10 +598,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -619,7 +619,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -663,10 +663,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -685,7 +685,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -818,7 +818,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -873,10 +873,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -897,7 +897,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1018,11 +1018,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1042,7 +1042,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1058,7 +1058,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1117,11 +1117,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1142,7 +1142,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1158,7 +1158,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/parallel.q.out b/ql/src/test/results/clientpositive/llap/parallel.q.out index 05220db..3c106f2 100644 --- a/ql/src/test/results/clientpositive/llap/parallel.q.out +++ b/ql/src/test/results/clientpositive/llap/parallel.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -66,7 +66,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -79,7 +79,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward diff --git a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out index 3254fb4..e785e39 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out @@ -124,10 +124,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -147,7 +147,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -182,10 +182,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -534,10 +534,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -555,7 +555,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -599,10 +599,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -621,7 +621,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -731,10 +731,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -754,7 +754,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -809,10 +809,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -833,7 +833,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -999,11 +999,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1023,7 +1023,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1039,7 +1039,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1098,11 +1098,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1123,7 +1123,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1139,7 +1139,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1210,11 +1210,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1235,7 +1235,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: int), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/partition_multilevels.q.out b/ql/src/test/results/clientpositive/llap/partition_multilevels.q.out index 30957b7..2bf9314 100644 --- a/ql/src/test/results/clientpositive/llap/partition_multilevels.q.out +++ b/ql/src/test/results/clientpositive/llap/partition_multilevels.q.out @@ -987,10 +987,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1013,7 +1013,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1589,10 +1589,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1615,7 +1615,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out b/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out index fa7abcb..0f2a1e8 100644 --- a/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out +++ b/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out @@ -147,11 +147,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -193,7 +195,7 @@ STAGE PLANS: MultiFileSpray: false Execution mode: llap LLAP IO: unknown - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -270,7 +272,50 @@ STAGE PLANS: name: default.t1_old Truncated Path -> Alias: /t1_old/ds=2011-10-13 [t1_old] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1, _col3 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: t1_mapping @@ -347,51 +392,6 @@ STAGE PLANS: name: default.t1_mapping Truncated Path -> Alias: /t1_mapping/ds=2011-10-13 [t1_mapping] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col3 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2 - columns.types string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -452,11 +452,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -544,7 +546,7 @@ STAGE PLANS: name: default.t1_new Truncated Path -> Alias: /t1_new/ds=2011-10-15 [t1_new] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -571,34 +573,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 5 - Map Operator Tree: - TableScan - alias: t1_mapping - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key (type: string), keymap (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -641,8 +616,33 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: t1_mapping + properties: + insideView TRUE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), keymap (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/ptf.q.out b/ql/src/test/results/clientpositive/llap/ptf.q.out index 542347d2..5f39957 100644 --- a/ql/src/test/results/clientpositive/llap/ptf.q.out +++ b/ql/src/test/results/clientpositive/llap/ptf.q.out @@ -33,11 +33,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -50,7 +50,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -77,7 +77,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -209,12 +209,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -230,22 +230,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -262,7 +247,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -289,7 +274,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -328,6 +313,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -402,10 +402,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -418,7 +418,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -528,11 +528,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -545,7 +545,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -572,7 +572,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -710,11 +710,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -727,7 +727,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -754,7 +754,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -895,11 +895,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -912,7 +912,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -947,7 +947,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1082,11 +1082,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1099,22 +1099,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1144,7 +1129,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1162,6 +1147,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1238,11 +1238,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1257,20 +1257,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_name (type: string) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1292,7 +1279,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_name (type: string) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1398,11 +1398,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1430,7 +1430,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1457,7 +1457,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1574,11 +1574,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1607,7 +1607,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1635,7 +1635,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1769,11 +1769,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1786,7 +1786,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1813,7 +1813,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1949,12 +1949,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1967,7 +1967,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2017,7 +2017,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2052,7 +2052,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2196,11 +2196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2213,7 +2213,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2240,7 +2240,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2380,12 +2380,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2398,22 +2398,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2443,7 +2428,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2460,7 +2445,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2525,6 +2510,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2609,11 +2609,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2626,7 +2626,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2661,7 +2661,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2767,11 +2767,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2794,7 +2794,7 @@ STAGE PLANS: value expressions: _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2823,7 +2823,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2995,13 +2995,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3014,7 +3014,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3047,7 +3047,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3100,7 +3100,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.part_4 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3137,7 +3137,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: sum_window_0 (type: bigint), _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3390,12 +3390,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3408,7 +3408,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3465,7 +3465,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3500,7 +3500,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3666,13 +3666,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3685,7 +3685,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3719,7 +3719,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3746,7 +3746,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3773,7 +3773,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3935,12 +3935,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3953,7 +3953,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3987,7 +3987,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4021,7 +4021,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4183,13 +4183,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4202,7 +4202,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4236,7 +4236,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4279,7 +4279,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4307,7 +4307,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4471,12 +4471,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4489,7 +4489,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4539,7 +4539,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4574,7 +4574,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4732,12 +4732,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4750,7 +4750,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4800,7 +4800,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4828,7 +4828,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out index b45e5ed..68477b5 100644 --- a/ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out @@ -71,10 +71,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -87,7 +87,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -201,10 +201,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -217,7 +217,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -321,10 +321,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -344,7 +344,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col5 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/ptf_streaming.q.out b/ql/src/test/results/clientpositive/llap/ptf_streaming.q.out index d410f5d..6b9101d 100644 --- a/ql/src/test/results/clientpositive/llap/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/llap/ptf_streaming.q.out @@ -33,11 +33,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -50,7 +50,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -77,7 +77,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -209,12 +209,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -230,22 +230,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -262,7 +247,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -289,7 +274,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -328,6 +313,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -404,11 +404,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -421,22 +421,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -466,7 +451,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -484,6 +469,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -560,11 +560,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -592,7 +592,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -619,7 +619,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -736,11 +736,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -769,7 +769,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,7 +797,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -933,12 +933,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -951,7 +951,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1001,7 +1001,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1036,7 +1036,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1174,12 +1174,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1192,7 +1192,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1242,7 +1242,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1277,7 +1277,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1415,12 +1415,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1433,7 +1433,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1483,7 +1483,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1518,7 +1518,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1660,12 +1660,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1678,22 +1678,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1723,7 +1708,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1740,7 +1725,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1805,6 +1790,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1911,12 +1911,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1929,7 +1929,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1986,7 +1986,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2021,7 +2021,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2187,13 +2187,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2206,7 +2206,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2240,7 +2240,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2267,7 +2267,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2294,7 +2294,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2454,12 +2454,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2472,7 +2472,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2522,7 +2522,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2550,7 +2550,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 219819a..bb5cd90 100644 --- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -98,7 +98,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -263,10 +263,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: complex_tbl_2 @@ -302,7 +302,7 @@ STAGE PLANS: tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) auto parallelism: true - Reducer 2 + Vertex 2 Needs Tagging: false Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out index 99a7119..c39668a 100644 --- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out +++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out @@ -21,10 +21,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -40,7 +40,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -80,10 +80,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +100,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -144,10 +144,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -164,7 +164,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -210,10 +210,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -236,7 +236,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -274,11 +274,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -298,26 +298,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -340,7 +321,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5580 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -360,6 +341,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -382,11 +382,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -406,26 +406,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -440,7 +421,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -454,6 +435,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -476,10 +476,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -500,7 +500,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -544,10 +544,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -568,7 +568,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2590,10 +2590,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2609,7 +2609,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2649,10 +2649,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2669,7 +2669,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2713,10 +2713,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2733,7 +2733,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2779,10 +2779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2798,7 +2798,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2836,11 +2836,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2860,26 +2860,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2895,7 +2876,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2915,6 +2896,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2937,11 +2937,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2961,26 +2961,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2995,7 +2976,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3009,6 +2990,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3031,10 +3031,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3050,7 +3050,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3094,10 +3094,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3113,7 +3113,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/sample1.q.out b/ql/src/test/results/clientpositive/llap/sample1.q.out index 979c5e5..85aff50 100644 --- a/ql/src/test/results/clientpositive/llap/sample1.q.out +++ b/ql/src/test/results/clientpositive/llap/sample1.q.out @@ -29,7 +29,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s diff --git a/ql/src/test/results/clientpositive/llap/sample10.q.out b/ql/src/test/results/clientpositive/llap/sample10.q.out index 3634b81..8249f57 100644 --- a/ql/src/test/results/clientpositive/llap/sample10.q.out +++ b/ql/src/test/results/clientpositive/llap/sample10.q.out @@ -52,11 +52,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpartbucket @@ -288,7 +288,7 @@ STAGE PLANS: /srcpartbucket/ds=2008-04-08/hr=12/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=11/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=12/000000_0 [srcpartbucket] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -306,7 +306,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out index 96134a3..7be4716 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out @@ -624,7 +624,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -819,7 +819,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out index 942fba7..46758fd 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out @@ -624,7 +624,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -819,7 +819,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out index 9e0dcc9..82610f5 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out @@ -92,12 +92,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=101) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_permute_select @@ -207,12 +202,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=145) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_string_permute_select @@ -388,12 +378,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=426) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -504,12 +489,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=586) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -697,12 +677,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=483) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -872,12 +847,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=756) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -1035,12 +1005,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=1113) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1232,12 +1197,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=236) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1359,12 +1319,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out index 744435a..6fb4601 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out @@ -162,12 +162,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=789) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -424,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=614) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -612,12 +602,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=293) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out index e0834bc..a4dc8ce 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out @@ -291,12 +291,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=1168) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -506,12 +501,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=1382) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -637,12 +627,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=494) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -752,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=407) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -948,12 +928,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=672) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out index 66778b1..38e625a 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out @@ -94,12 +94,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=99) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_permute_select @@ -217,12 +212,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=99) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_string_permute_select @@ -410,12 +400,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=422) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=5 width=422) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -694,12 +679,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=164) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=5 width=164) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -865,12 +845,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=588) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=5 width=588) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out index 07aac08..af62fbb 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out @@ -92,12 +92,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=101) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_permute_select @@ -207,12 +202,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=145) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_string_permute_select @@ -388,12 +378,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=426) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -504,12 +489,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=586) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -697,12 +677,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=483) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -872,12 +847,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=756) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -1035,12 +1005,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=1113) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1232,12 +1197,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=236) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1359,12 +1319,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out index 255dc58..11d5c55 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out @@ -162,12 +162,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=789) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -424,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=614) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -612,12 +602,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=293) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out index e1496bb..4cc22bc 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out @@ -291,12 +291,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=1168) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -506,12 +501,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=1382) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -637,12 +627,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=494) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -752,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=407) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -948,12 +928,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=672) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out index e67c613..a4a78c1 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out @@ -94,12 +94,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=99) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_permute_select @@ -217,12 +212,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=99) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_string_permute_select @@ -410,12 +400,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=422) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=5 width=422) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -694,12 +679,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=164) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=5 width=164) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -865,12 +845,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=588) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=5 width=588) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out index bf7d9aa..3767842 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out @@ -92,12 +92,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_permute_select @@ -207,12 +202,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_string_permute_select @@ -388,12 +378,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -504,12 +489,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -697,12 +677,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -872,12 +847,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -1035,12 +1005,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1232,12 +1197,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1359,12 +1319,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out index 803fc66..8408832 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out @@ -162,12 +162,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -424,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -612,12 +602,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out index 2a2e48a..a7de5ce 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out @@ -291,12 +291,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=512) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -506,12 +501,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=455) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -637,12 +627,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=151) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -752,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -948,12 +928,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out index f3be02f..fec61b3 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out @@ -94,12 +94,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_permute_select @@ -217,12 +212,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_string_permute_select @@ -410,12 +400,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=90) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=5 width=90) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -694,12 +679,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=151) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=5 width=151) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -865,12 +845,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=250) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=5 width=250) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out index 6632bfb..b2a943c 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out @@ -96,12 +96,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_permute_select @@ -211,12 +206,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_string_permute_select @@ -392,12 +382,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=56) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -508,12 +493,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=154) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -701,12 +681,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=153) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -876,12 +851,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=231) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -1039,12 +1009,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=70) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1236,12 +1201,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1363,12 +1323,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out index 3c15417..42e43c1 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out @@ -166,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -428,12 +423,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -616,12 +606,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out index 456f83d..47bbebc 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out @@ -295,12 +295,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=507) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -510,12 +505,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=453) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -641,12 +631,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=150) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -756,12 +741,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -952,12 +932,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out index 5206770..b3203fa 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out @@ -98,12 +98,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_permute_select @@ -221,12 +216,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_string_permute_select @@ -414,12 +404,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=90) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=5 width=90) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -698,12 +683,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=151) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=5 width=151) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -869,12 +849,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=250) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=5 width=250) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out index 0298a29..951e9db 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out @@ -96,12 +96,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_permute_select @@ -211,12 +206,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,part,a,b from part_add_int_string_permute_select @@ -392,12 +382,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -508,12 +493,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -701,12 +681,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -876,12 +851,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -1039,12 +1009,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1236,12 +1201,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1363,12 +1323,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out index 1b5c6bf..c96df99 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out @@ -168,12 +168,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -430,12 +425,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -618,12 +608,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out index 4787841..a63db40 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out @@ -295,12 +295,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=512) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -510,12 +505,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=455) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -641,12 +631,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=151) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -756,12 +741,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -952,12 +932,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out index 5c59620..e85eb2a 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out @@ -98,12 +98,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_permute_select @@ -221,12 +216,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=5 width=20) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right select insert_num,a,b from table_add_int_string_permute_select @@ -414,12 +404,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=90) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=5 width=90) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -698,12 +683,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=151) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=5 width=151) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -869,12 +849,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=250) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=5 width=250) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out b/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out index 2eff533..1da5570 100644 --- a/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out +++ b/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out @@ -15,10 +15,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -39,7 +39,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -712,10 +712,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -735,7 +735,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -936,12 +936,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -962,7 +964,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -983,23 +1000,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -2477,10 +2477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2501,7 +2501,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3170,10 +3170,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3193,7 +3193,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3394,12 +3394,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3420,7 +3422,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -3441,23 +3458,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/semijoin.q.out b/ql/src/test/results/clientpositive/llap/semijoin.q.out index ad43098..91a5569 100644 --- a/ql/src/test/results/clientpositive/llap/semijoin.q.out +++ b/ql/src/test/results/clientpositive/llap/semijoin.q.out @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -157,31 +157,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -196,7 +172,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -210,6 +186,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -246,11 +246,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -266,31 +266,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -305,7 +281,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -319,6 +295,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -377,31 +377,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -416,7 +392,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -430,6 +406,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -460,11 +460,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -480,31 +480,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 15) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -523,7 +499,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -537,6 +513,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 15) (type: boolean) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -578,11 +578,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -598,31 +598,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value < 'val_10') and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -637,7 +613,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -651,6 +627,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value < 'val_10') and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -684,11 +684,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -712,23 +712,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -747,7 +731,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -761,17 +745,33 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t3 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t3 #### A masked pattern was here #### POSTHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value POSTHOOK: type: QUERY @@ -794,11 +794,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -822,23 +822,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -857,7 +841,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -871,6 +855,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -901,11 +901,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -929,23 +929,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -960,7 +944,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -974,6 +958,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1009,11 +1009,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1028,31 +1028,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1067,7 +1043,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1081,6 +1057,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1130,11 +1130,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1150,31 +1150,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (2 * key) is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (2 * _col0) (type: int) - sort order: + - Map-reduce partition columns: (2 * _col0) (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1189,7 +1165,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1203,6 +1179,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (2 * key) is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (2 * _col0) (type: int) + sort order: + + Map-reduce partition columns: (2 * _col0) (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1237,11 +1237,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1257,7 +1257,43 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1273,7 +1309,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1297,42 +1333,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: int), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1379,11 +1379,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1398,31 +1398,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1437,7 +1413,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1451,6 +1427,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1495,11 +1495,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1514,7 +1514,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Semi Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1538,7 +1569,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1562,37 +1593,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Semi Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1639,11 +1639,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1655,7 +1655,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1667,7 +1698,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1688,37 +1719,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1777,11 +1777,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1793,7 +1793,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1805,7 +1836,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1826,37 +1857,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1918,11 +1918,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1934,7 +1934,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1946,7 +1977,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1967,37 +1998,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2059,56 +2059,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2125,7 +2092,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2139,6 +2106,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2200,11 +2200,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2216,7 +2216,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2237,7 +2268,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2249,37 +2280,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2343,11 +2343,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2359,7 +2359,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2380,7 +2411,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2392,37 +2423,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2497,12 +2497,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2518,43 +2518,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2571,7 +2535,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2586,7 +2550,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 196 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2600,6 +2564,42 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2667,10 +2667,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2690,7 +2690,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2714,24 +2732,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/skewjoin.q.out b/ql/src/test/results/clientpositive/llap/skewjoin.q.out index 617045c..07820a8 100644 --- a/ql/src/test/results/clientpositive/llap/skewjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/skewjoin.q.out @@ -93,10 +93,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -115,27 +115,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -158,6 +138,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -217,10 +217,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -236,7 +236,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -252,7 +278,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -268,7 +294,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -284,32 +310,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -359,10 +359,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -378,7 +378,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -394,7 +420,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -410,7 +436,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -426,32 +452,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -493,11 +493,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -513,23 +513,7 @@ STAGE PLANS: value expressions: key (type: string), val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: key (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -549,7 +533,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -564,6 +548,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: key (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -605,11 +605,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -628,27 +628,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -672,7 +652,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -687,6 +667,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -736,11 +736,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -759,27 +759,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -803,7 +783,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -818,6 +798,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -875,11 +875,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -898,46 +898,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -963,7 +924,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -978,6 +939,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1025,11 +1025,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: k @@ -1042,20 +1042,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: v - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1075,7 +1062,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1090,6 +1077,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: v + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out b/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out index 2c255ed..758d2c4 100644 --- a/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out +++ b/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out @@ -95,10 +95,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -118,7 +118,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -138,24 +156,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -200,10 +200,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -220,24 +220,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -255,6 +238,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -301,11 +301,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -324,26 +324,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -362,7 +343,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -377,6 +358,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -410,11 +410,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -430,23 +430,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -465,7 +449,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -480,6 +464,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_cache.q.out b/ql/src/test/results/clientpositive/llap/smb_cache.q.out index 932b211..85b17f0 100644 --- a/ql/src/test/results/clientpositive/llap/smb_cache.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_cache.q.out @@ -203,34 +203,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, @@ -283,34 +262,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out index cafdf5d..84ad910 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out @@ -59,11 +59,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -78,22 +78,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -112,7 +97,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -127,6 +112,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -178,12 +178,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -198,22 +198,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -236,7 +221,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -250,7 +235,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -264,6 +249,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -328,12 +328,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -348,22 +348,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -386,7 +371,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -406,7 +391,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -421,6 +406,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -480,30 +480,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -522,7 +503,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -541,7 +522,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -556,6 +537,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -619,11 +619,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -642,22 +642,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -676,7 +661,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -691,6 +676,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -778,30 +778,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -820,7 +801,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -839,7 +820,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -854,6 +835,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 8) and (key < 6)) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -931,11 +931,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -954,26 +954,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -992,7 +973,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1007,6 +988,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1060,11 +1060,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1083,26 +1083,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1121,7 +1102,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1136,6 +1117,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1185,11 +1185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1208,22 +1208,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1242,7 +1227,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1257,6 +1242,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1302,11 +1302,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1325,22 +1325,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1359,7 +1344,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1374,6 +1359,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1429,11 +1429,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1452,7 +1452,43 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1471,7 +1507,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -1490,42 +1526,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1597,11 +1597,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1620,22 +1620,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1654,7 +1639,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1669,6 +1654,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out index daf2497..f632173 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -135,7 +135,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -209,66 +269,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types int:string:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -367,11 +367,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -445,7 +445,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -519,66 +579,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -625,11 +625,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -703,7 +703,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key2 (type: int), key (type: int) + 1 key2 (type: int), key (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -777,66 +837,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key2 (type: int), key (type: int) - 1 key2 (type: int), key (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -883,11 +883,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -961,7 +961,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1035,66 +1095,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), value (type: string) - 1 key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out index c69be29..3f99f1f 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out @@ -189,11 +189,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -208,7 +208,51 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + Inner Join 0 to 6 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + 3 key (type: int) + 4 key (type: int) + 5 key (type: int) + 6 key (type: int) + Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -223,7 +267,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -238,7 +282,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: d @@ -253,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: e @@ -268,7 +312,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: f @@ -283,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: g @@ -298,50 +342,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - Inner Join 0 to 4 - Inner Join 0 to 5 - Inner Join 0 to 6 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - 3 key (type: int) - 4 key (type: int) - 5 key (type: int) - 6 key (type: int) - Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -413,10 +413,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -500,7 +500,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -584,10 +584,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -681,7 +681,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -797,10 +797,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -972,7 +972,7 @@ STAGE PLANS: Statistics: Num rows: 165 Data size: 1155 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: llap - Map 18 + Vertex 18 Map Operator Tree: TableScan alias: q @@ -988,7 +988,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 19 + Vertex 19 Map Operator Tree: TableScan alias: r @@ -1004,7 +1004,31 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 20 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join0 to 2 + Left Outer Join0 to 3 + Left Outer Join0 to 4 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 20 Map Operator Tree: TableScan alias: s @@ -1020,7 +1044,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 21 + Vertex 21 Map Operator Tree: TableScan alias: t @@ -1036,30 +1060,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join0 to 2 - Left Outer Join0 to 3 - Left Outer Join0 to 4 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out index c29a36b..e1739f5 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out @@ -55,7 +55,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -238,10 +238,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -261,7 +261,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -381,7 +381,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index 1b53fdb..8f805d7 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -55,7 +55,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out index 4dd7f4b..f7e9777 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out @@ -65,10 +65,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -84,7 +84,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -100,7 +124,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -116,30 +140,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -174,10 +174,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -190,7 +190,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -203,7 +227,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -216,30 +240,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -274,10 +274,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -290,7 +290,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -303,7 +327,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -316,30 +340,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -379,10 +379,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -395,7 +395,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -408,7 +432,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -421,30 +445,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -485,10 +485,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -501,7 +501,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -514,7 +538,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -527,30 +551,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -612,7 +612,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -625,7 +649,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -638,30 +662,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -698,10 +698,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -714,7 +714,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -727,7 +751,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -740,30 +764,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -802,10 +802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -814,37 +814,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -868,6 +842,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -908,10 +908,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -924,7 +924,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -937,7 +961,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -950,30 +974,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1016,10 +1016,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1032,7 +1032,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1045,7 +1069,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1058,30 +1082,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1118,10 +1118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1134,7 +1134,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1147,7 +1171,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1160,30 +1184,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1227,10 +1227,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1243,7 +1243,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1256,7 +1280,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1269,30 +1293,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1333,10 +1333,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1349,7 +1349,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1362,7 +1386,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1375,30 +1399,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out index 152c3e0..7c4cca9 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out @@ -65,10 +65,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -84,7 +84,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -100,7 +124,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -116,30 +140,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -174,10 +174,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -190,7 +190,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -203,7 +227,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -216,30 +240,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -274,10 +274,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -290,7 +290,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -303,7 +327,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -316,30 +340,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -379,10 +379,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -395,7 +395,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -408,7 +432,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -421,30 +445,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -485,10 +485,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -501,7 +501,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -514,7 +538,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -527,30 +551,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -612,7 +612,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -625,7 +649,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -638,30 +662,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -698,10 +698,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -714,7 +714,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -727,7 +751,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -740,30 +764,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -802,10 +802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -814,37 +814,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -868,6 +842,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -908,10 +908,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -924,7 +924,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -937,7 +961,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -950,30 +974,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1016,10 +1016,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1032,7 +1032,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1045,7 +1069,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1058,30 +1082,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1118,10 +1118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1134,7 +1134,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1147,7 +1171,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1160,30 +1184,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1227,10 +1227,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1243,7 +1243,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1256,7 +1280,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1269,30 +1293,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1333,10 +1333,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1349,7 +1349,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1362,7 +1386,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1375,30 +1399,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out index bbf9b46..6440616 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -92,23 +92,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,6 +115,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -1251,10 +1251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1270,23 +1270,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1309,6 +1293,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2445,10 +2445,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2464,23 +2464,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2503,6 +2487,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2555,10 +2555,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2574,23 +2574,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2613,6 +2597,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2661,10 +2661,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2680,7 +2680,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2696,7 +2720,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -2712,30 +2736,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/stats11.q.out b/ql/src/test/results/clientpositive/llap/stats11.q.out index b4643b1..4db4ee0 100644 --- a/ql/src/test/results/clientpositive/llap/stats11.q.out +++ b/ql/src/test/results/clientpositive/llap/stats11.q.out @@ -313,10 +313,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -386,7 +386,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -458,54 +506,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -738,7 +738,55 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -810,54 +858,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out b/ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out index 8a85bf4..1bff6aa 100644 --- a/ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out @@ -101,7 +101,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -142,7 +142,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git a/ql/src/test/results/clientpositive/llap/stats_only_null.q.out b/ql/src/test/results/clientpositive/llap/stats_only_null.q.out index d59d4ac..de80baa 100644 --- a/ql/src/test/results/clientpositive/llap/stats_only_null.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_only_null.q.out @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -141,10 +141,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null_part @@ -164,7 +164,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out index 3d8251f..08e1e4e 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out @@ -33,13 +33,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -55,7 +55,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -74,42 +92,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -130,7 +113,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -143,7 +126,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -303,13 +303,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -326,40 +326,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -377,7 +344,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -398,7 +381,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -411,7 +394,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -431,7 +431,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- uncorr exists explain select * @@ -461,11 +461,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -480,7 +480,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -502,25 +520,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -540,7 +540,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src b where exists diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index 887a27e..f0a2420 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -44,7 +44,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s1 @@ -64,25 +82,7 @@ STAGE PLANS: Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -154,13 +154,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -176,7 +176,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -196,42 +214,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -252,7 +235,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -265,7 +248,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -345,13 +345,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -368,20 +368,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -399,7 +386,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -443,7 +443,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -461,7 +461,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -530,15 +530,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -555,37 +555,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -607,7 +577,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -648,7 +631,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -675,7 +658,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -694,7 +677,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -711,7 +694,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -784,13 +784,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -806,7 +806,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -826,42 +844,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -882,7 +865,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -900,7 +883,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -996,13 +996,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1022,51 +1022,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: l_orderkey - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: l_orderkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1079,7 +1035,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1096,7 +1052,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1118,7 +1074,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int) + outputColumnNames: l_orderkey + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: l_orderkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1180,7 +1180,7 @@ POSTHOOK: Input: default@lineitem #### A masked pattern was here #### 108570 8571 4297 1798 -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: -- corr, agg in outer and inner explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY @@ -1196,17 +1196,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1226,7 +1226,7 @@ STAGE PLANS: value expressions: _col1 (type: double), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: lineitem @@ -1238,58 +1238,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1310,7 +1259,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1323,7 +1272,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1340,7 +1304,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: double), _col3 (type: int) Statistics: Num rows: 26 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1360,7 +1324,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1375,7 +1339,43 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1402,7 +1402,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1421,7 +1421,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1441,7 +1441,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@lineitem @@ -1468,14 +1468,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1495,41 +1495,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1547,7 +1513,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1570,7 +1553,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1272 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1597,7 +1580,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1614,7 +1597,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1680,12 +1680,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1702,7 +1702,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col5 - 1) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1725,25 +1743,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col5 - 1) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1766,7 +1766,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1817,12 +1817,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1839,7 +1839,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col0 * _col5) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1862,25 +1880,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col0 * _col5) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1903,7 +1903,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1947,14 +1947,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -1970,41 +1970,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2023,7 +1989,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2038,7 +2004,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2063,7 +2046,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2080,7 +2063,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2124,12 +2124,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2146,7 +2146,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 floor(_col7) (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -2169,25 +2187,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 floor(_col7) (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2210,7 +2210,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2276,13 +2276,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2299,41 +2299,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2351,7 +2317,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2372,7 +2355,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2389,7 +2372,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2434,13 +2434,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2457,41 +2457,7 @@ STAGE PLANS: value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2509,7 +2475,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2534,7 +2517,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2551,7 +2534,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2620,13 +2620,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2643,41 +2643,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2695,7 +2661,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2716,7 +2699,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2729,7 +2712,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2798,13 +2798,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2821,45 +2821,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2877,7 +2839,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2898,7 +2877,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2911,7 +2890,28 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2980,15 +2980,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3005,48 +3005,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3069,7 +3028,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3084,7 +3043,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3102,7 +3061,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3115,7 +3091,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3138,7 +3138,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3212,17 +3212,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3238,7 +3238,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -3262,41 +3262,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3322,7 +3288,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3335,7 +3301,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3358,7 +3324,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3376,7 +3342,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3394,7 +3360,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3419,7 +3402,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3436,7 +3419,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3510,13 +3510,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3533,30 +3533,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3572,7 +3549,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3586,7 +3563,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3609,7 +3609,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3662,13 +3662,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3685,30 +3685,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3725,7 +3702,7 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3742,7 +3719,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3765,7 +3765,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3812,13 +3812,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Union 4 (CONTAINS) - Map 6 <- Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3835,7 +3837,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -3856,7 +3876,20 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 6 Map Operator Tree: TableScan alias: part @@ -3877,39 +3910,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -3944,14 +3944,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -3968,44 +3968,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_size < 10) (type: boolean) - Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_mfgr (type: string), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4025,7 +3988,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4039,7 +4002,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_size < 10) (type: boolean) + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_mfgr (type: string), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4064,7 +4047,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4081,7 +4064,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4134,14 +4134,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4158,7 +4158,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4178,62 +4196,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4250,7 +4213,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4275,7 +4238,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4288,7 +4251,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4301,6 +4281,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4359,16 +4359,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4385,44 +4385,24 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) + Vertex 11 Map Operator Tree: TableScan alias: part @@ -4439,44 +4419,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4489,7 +4432,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4507,7 +4450,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4524,7 +4487,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 18 Data size: 4050 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4545,7 +4508,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4562,7 +4525,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 8 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4575,6 +4555,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4635,15 +4635,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4660,7 +4660,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4681,64 +4681,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4756,7 +4699,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4774,7 +4717,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4797,7 +4760,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4810,7 +4773,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4827,7 +4807,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4868,14 +4868,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4892,7 +4892,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4912,62 +4930,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4984,7 +4947,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5009,7 +4972,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5022,7 +4985,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5035,6 +5015,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 7765221..5390b52 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -94,13 +94,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -117,41 +117,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -168,7 +134,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -186,7 +152,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -199,7 +182,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -254,7 +254,7 @@ POSTHOOK: Input: default@part_null 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- NOT IN has null value so should return 0 rows explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY @@ -270,15 +270,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -295,65 +295,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -367,7 +309,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -382,7 +324,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -399,7 +341,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -424,7 +366,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -437,7 +396,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -449,6 +428,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -456,7 +456,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -465,7 +465,7 @@ POSTHOOK: query: select * from part_null where p_name IN (select p_name from par POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- NOT IN is always true and IN is false for where p_name is NULL, hence should return all but one row explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY @@ -481,15 +481,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -506,65 +506,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_type) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -578,7 +520,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -593,7 +535,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -610,7 +552,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -635,7 +577,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -648,18 +607,59 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_type) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -667,7 +667,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -702,7 +702,7 @@ POSTHOOK: Input: default@part_null 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- NOT IN has one NULL value so this whole query should not return any row explain select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY @@ -718,15 +718,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -743,65 +743,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -815,7 +757,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -830,7 +772,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -847,7 +789,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -872,7 +814,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -885,7 +844,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -897,6 +876,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -904,7 +904,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -913,7 +913,7 @@ POSTHOOK: query: select * from part_null where p_brand IN (select p_brand from p POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- NOT IN is always true irrespective of p_name being null/non-null since inner query is empty -- second query is always true so this should return all rows explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) @@ -931,15 +931,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -954,65 +954,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: c - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count(), count(c) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: _col0 (type: char(2)), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(2)), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: char(2)), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1025,7 +967,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1042,7 +984,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 5 Data size: 3341 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1066,7 +1008,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 2 Data size: 1470 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1084,7 +1026,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c (type: char(2)) + outputColumnNames: c + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count(), count(c) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1096,7 +1058,28 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c (type: char(2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: _col0 (type: char(2)), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(2)), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: char(2)), _col1 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1110,14 +1093,31 @@ STAGE PLANS: Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: boolean) - - Stage: Stage-0 + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1155,7 +1155,7 @@ POSTHOOK: Input: default@tempty 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: -- IN, EXISTS explain select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY @@ -1171,13 +1171,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1194,43 +1194,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tnull - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1245,7 +1209,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1263,7 +1227,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1276,7 +1257,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tnull + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1296,7 +1296,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1333,7 +1333,7 @@ POSTHOOK: Input: default@tnull 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) @@ -1347,13 +1347,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1370,43 +1370,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1421,7 +1385,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1439,7 +1403,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1452,7 +1433,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1472,7 +1472,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1483,7 +1483,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null POSTHOOK: Input: default@tempty #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) @@ -1497,13 +1497,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1520,43 +1520,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1571,7 +1535,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1596,7 +1560,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1609,7 +1590,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1628,7 +1628,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1665,8 +1665,8 @@ POSTHOOK: Input: default@tempty 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: -- corr, mix of IN/NOT IN explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY @@ -1682,35 +1682,35 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1727,7 +1727,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -1744,215 +1744,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1969,7 +1761,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1996,7 +1788,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2011,7 +1803,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2036,7 +1845,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2053,7 +1862,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2074,7 +1898,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2087,7 +1911,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2104,7 +1928,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2117,7 +1962,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2138,7 +1998,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2151,7 +2011,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2164,7 +2045,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2181,7 +2079,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2202,7 +2100,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2219,7 +2117,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 6 Data size: 1633 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2237,7 +2135,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2254,7 +2152,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2279,7 +2194,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2296,7 +2211,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2309,7 +2241,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2322,7 +2271,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2347,7 +2313,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2360,7 +2326,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2385,7 +2368,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2402,7 +2385,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2422,8 +2422,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2475,17 +2475,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -2505,84 +2505,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2603,7 +2526,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 17 Data size: 3581 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2616,7 +2539,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1685 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2629,7 +2572,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2646,7 +2589,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2664,7 +2607,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2677,7 +2640,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2698,7 +2678,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 5605 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2711,6 +2691,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 2802 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2754,8 +2754,8 @@ POSTHOOK: Input: default@part_null 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: -- one query has multiple corr explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY @@ -2767,162 +2767,56 @@ STAGE DEPENDENCIES: Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) - sort order: +++ - Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + alias: part_null + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) + sort order: +++ + Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 27 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2939,109 +2833,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3058,7 +2850,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3085,7 +2877,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3100,7 +2892,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3125,7 +2934,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3142,7 +2951,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3163,7 +2987,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3176,7 +3000,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3193,7 +3017,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 5 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3206,7 +3051,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3227,7 +3087,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3240,7 +3100,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3253,7 +3134,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3270,7 +3168,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3291,7 +3189,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3308,7 +3206,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col3 (type: string) Statistics: Num rows: 5 Data size: 3939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col13 (type: bigint), _col14 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3326,7 +3224,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3343,7 +3241,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3368,7 +3283,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3385,7 +3300,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3398,7 +3330,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3411,7 +3360,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3436,7 +3402,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3449,7 +3415,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3474,7 +3457,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3491,7 +3474,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3511,8 +3511,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3564,23 +3564,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 22 <- Map 21 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -3597,41 +3597,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -3648,75 +3614,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3743,7 +3641,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3758,7 +3656,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3771,7 +3686,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3792,7 +3724,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3810,7 +3742,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3827,7 +3759,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3844,7 +3793,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: int) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3857,7 +3806,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 22 + Vertex 21 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 22 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3870,7 +3836,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3887,7 +3853,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col5 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3912,7 +3878,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3933,7 +3916,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 6435 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3946,7 +3929,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4019,17 +4019,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -4046,7 +4046,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4063,7 +4063,20 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 12 Map Operator Tree: TableScan alias: part @@ -4080,41 +4093,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col1 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4127,20 +4106,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4158,7 +4124,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col1 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4175,7 +4158,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4196,7 +4179,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4209,7 +4192,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4234,7 +4234,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4294,7 +4294,7 @@ POSTHOOK: Input: default@part_null 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: -- in, not in corr explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey @@ -4316,22 +4316,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Map 15 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 8 <- Reducer 17 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 20 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -4351,7 +4351,20 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: lineitem @@ -4371,120 +4384,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4497,7 +4397,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4518,7 +4418,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4531,7 +4431,47 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 15 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4548,7 +4488,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4566,7 +4526,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4579,7 +4539,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4597,7 +4557,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4614,7 +4574,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: int) Statistics: Num rows: 5 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4636,7 +4596,47 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4651,7 +4651,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: double), _col4 (type: int) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4668,7 +4668,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: double) Statistics: Num rows: 14 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col4 (type: int), _col5 (type: bigint), _col6 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4703,7 +4703,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4743,21 +4743,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -4773,7 +4773,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -4793,81 +4793,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4890,7 +4816,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4916,7 +4842,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4929,7 +4855,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4950,7 +4892,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4963,7 +4905,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4976,7 +4942,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 6552 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4999,7 +4965,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5017,7 +4983,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: bigint) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5035,7 +5001,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5056,7 +5039,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5069,7 +5052,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5404,7 +5404,7 @@ POSTHOOK: Input: default@src 431 val_431 3 430 val_430 3 417 val_417 3 -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: -- subquery pred only refer to parent query column explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY @@ -5420,13 +5420,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5443,42 +5443,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_type <> '1') (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5496,7 +5461,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5517,7 +5497,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5530,7 +5510,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_type <> '1') (type: boolean) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5549,7 +5549,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index eb99650..0733088 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- non agg, non corr explain select * @@ -26,13 +26,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -47,50 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(key) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -107,7 +64,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -132,7 +89,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(key) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -144,7 +120,31 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -165,7 +165,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where src.key not in ( select key from src s1 where s1.key > '2') @@ -328,21 +328,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -359,55 +359,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: p_name (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -421,24 +373,7 @@ STAGE PLANS: value expressions: p_name (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -479,7 +414,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -500,7 +435,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -518,7 +453,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -535,7 +470,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -548,7 +500,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -561,7 +530,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -578,7 +547,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -603,7 +572,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: p_name (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -644,7 +627,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -671,7 +654,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -686,7 +669,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -744,7 +744,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- agg, non corr explain select p_name, p_size @@ -774,16 +774,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -798,33 +798,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -838,7 +812,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -855,7 +829,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col1) (type: double) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -880,7 +854,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -924,7 +911,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -941,7 +928,20 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -985,7 +985,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1010,7 +1010,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -1084,22 +1084,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1116,7 +1116,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -1129,71 +1129,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1234,7 +1170,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1261,7 +1197,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1280,7 +1216,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1298,7 +1234,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 15 + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1315,7 +1251,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1328,7 +1281,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1341,7 +1311,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1358,7 +1328,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 26 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1383,7 +1353,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1424,7 +1407,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1451,7 +1434,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1472,7 +1455,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1532,7 +1532,7 @@ Manufacturer#5 almond antique medium spring khaki 6 Manufacturer#5 almond azure blanched chiffon midnight 23 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- non agg, non corr, Group By in Parent Query select li.l_partkey, count(*) from lineitem li @@ -1567,7 +1567,7 @@ POSTHOOK: Input: default@lineitem 139636 1 175839 1 182052 1 -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- alternate not in syntax select * from src @@ -1731,7 +1731,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1749,13 +1749,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1775,58 +1775,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 12972 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 69 Data size: 12972 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1843,7 +1792,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1868,7 +1817,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col0) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1880,7 +1854,33 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 69 Data size: 12972 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 69 Data size: 12972 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1901,7 +1901,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY @@ -1931,20 +1931,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1964,113 +1964,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) - Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2093,7 +1987,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1272 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2120,7 +2014,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2138,7 +2032,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2155,7 +2049,31 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2168,7 +2086,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) + Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_size (type: int) + outputColumnNames: p_size + Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2181,7 +2123,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2198,7 +2140,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2223,7 +2165,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2246,7 +2205,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 1272 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2271,7 +2230,31 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((p_brand <> 'Brand#14') and (p_size <> 340)) (type: boolean) + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 5200 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2284,6 +2267,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2300,7 +2300,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: --lhs contains non-simple expression explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY @@ -2316,15 +2316,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2339,53 +2339,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2402,7 +2356,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2427,7 +2381,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2449,7 +2426,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2461,7 +2438,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2488,7 +2488,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2509,7 +2509,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2534,7 +2534,7 @@ POSTHOOK: Input: default@part 48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) @@ -2548,15 +2548,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2571,53 +2571,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2634,7 +2588,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 * _col5) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2659,7 +2613,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2681,7 +2658,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2693,7 +2670,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2720,7 +2720,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2741,7 +2741,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2790,20 +2790,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -2820,58 +2820,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2888,24 +2837,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2926,7 +2858,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2944,7 +2876,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2961,7 +2893,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2974,7 +2923,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2988,7 +2954,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 + 100) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3005,7 +2971,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3030,7 +2996,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3045,7 +3011,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3072,7 +3055,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3087,7 +3070,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3116,7 +3116,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 26 -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- lhs contains udf expression explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY @@ -3132,15 +3132,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3155,53 +3155,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3218,7 +3172,7 @@ STAGE PLANS: Map-reduce partition columns: floor(_col7) (type: bigint) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3243,11 +3197,34 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 @@ -3265,7 +3242,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3277,7 +3254,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3300,7 +3300,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: boolean) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3321,7 +3321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3344,19 +3344,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3373,92 +3373,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3479,7 +3394,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3497,7 +3412,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3514,7 +3429,24 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3527,7 +3459,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3540,7 +3489,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3557,7 +3506,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col5 (type: int), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3582,7 +3531,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3609,7 +3575,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3628,7 +3594,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3641,6 +3624,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3695,19 +3695,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3724,113 +3724,28 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int), _col2 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col1, _col3, _col4 - Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: string), _col3 (type: int), _col4 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int) - Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int), _col2 (type: int) + 1 _col0 (type: int), _col1 (type: int) + outputColumnNames: _col1, _col3, _col4 + Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: string), _col3 (type: int), _col4 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int) + Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3848,7 +3763,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3865,7 +3780,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3878,7 +3810,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3891,7 +3840,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3908,7 +3857,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3933,7 +3882,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3960,7 +3926,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3975,7 +3941,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3988,6 +3971,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4018,19 +4018,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4047,92 +4047,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4153,7 +4068,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4171,7 +4086,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4188,7 +4103,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4201,7 +4133,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4214,7 +4163,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4231,7 +4180,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: string) Statistics: Num rows: 26 Data size: 8658 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4256,7 +4205,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4283,7 +4249,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4298,7 +4264,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4311,6 +4294,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4341,19 +4341,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4370,100 +4370,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4484,7 +4391,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4502,7 +4409,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 9 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4519,7 +4426,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4532,7 +4460,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4545,7 +4490,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4562,7 +4507,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 26 Data size: 6370 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4587,7 +4532,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4614,7 +4576,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4629,7 +4591,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4642,6 +4625,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4657,7 +4657,7 @@ POSTHOOK: query: select p_name from (select p_name, p_type, p_size+1 as size fro POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- where plus having explain select key, count(*) from src where value NOT IN (select value from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY @@ -4673,17 +4673,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -4694,76 +4694,35 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(value) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 10 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 214 Data size: 20330 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 214 Data size: 20330 Basic stats: COMPLETE Column stats: COMPLETE + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4786,7 +4745,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4799,7 +4758,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4816,7 +4775,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4846,7 +4805,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4861,7 +4820,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4879,7 +4838,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(value) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4891,7 +4870,28 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 214 Data size: 20330 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 214 Data size: 20330 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4912,7 +4912,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select value from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -4936,23 +4936,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 21 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -4969,82 +4969,20 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: sc @@ -5061,37 +4999,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5112,7 +5020,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5130,7 +5038,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5147,7 +5055,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5160,7 +5085,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5173,7 +5115,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5190,7 +5156,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5216,7 +5182,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5229,7 +5195,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5259,7 +5225,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5277,7 +5243,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5295,7 +5261,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5322,7 +5305,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5337,6 +5320,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5352,7 +5352,7 @@ POSTHOOK: query: select key, count(*) from src where value NOT IN (select value POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- subquery with order by explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY @@ -5368,16 +5368,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5392,53 +5392,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5452,7 +5406,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5469,7 +5423,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5492,7 +5446,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -5506,7 +5460,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5528,7 +5505,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5540,7 +5517,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5574,7 +5574,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5599,7 +5599,7 @@ POSTHOOK: Input: default@part 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: --order by with limit explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand limit 4 PREHOOK: type: QUERY @@ -5615,16 +5615,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5639,53 +5639,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5699,7 +5653,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5716,7 +5670,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5740,7 +5694,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -5757,7 +5711,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5779,7 +5756,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5791,7 +5768,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5825,7 +5825,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5834,11 +5834,11 @@ POSTHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- union, uncorr explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY @@ -5854,17 +5854,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 11 <- Union 9 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 6 <- Union 5 (SIMPLE_EDGE) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 9 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) + Vertex 8 <- Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 5 + Vertex: Union 5 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -5879,76 +5883,29 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: boolean) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) + Vertex 11 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: p_name (type: string) + expressions: p_brand (type: string) outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), true (type: boolean) mode: hash @@ -5961,21 +5918,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: boolean) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5992,7 +5935,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6017,7 +5960,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col0) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6029,10 +5992,47 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Union 5 - Vertex: Union 5 - Union 9 - Vertex: Union 9 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col0) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -6040,7 +6040,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6564,20 +6564,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -6594,58 +6594,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -6662,24 +6611,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6700,7 +6632,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6718,7 +6650,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6735,7 +6667,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6748,7 +6697,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6762,7 +6728,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble((_col0 + 100)) (type: double) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6779,7 +6745,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6804,7 +6770,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6819,7 +6785,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6846,7 +6829,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6861,7 +6844,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6923,7 +6923,7 @@ POSTHOOK: query: INSERT INTO t2 VALUES (null), (2), (100) POSTHOOK: type: QUERY POSTHOOK: Output: default@t2 POSTHOOK: Lineage: t2.c1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- uncorr explain SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY @@ -6939,69 +6939,28 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(c1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7018,7 +6977,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 381 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7043,7 +7002,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: c1 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(c1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7055,7 +7034,28 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7076,7 +7076,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY PREHOOK: Input: default@t1 @@ -7102,19 +7102,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7131,92 +7131,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7237,7 +7152,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7255,7 +7170,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: char(100)), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7272,7 +7187,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: char(100)) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7286,7 +7218,24 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7299,7 +7248,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7316,7 +7265,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7341,7 +7290,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7364,7 +7330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7379,7 +7345,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7393,6 +7376,23 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7471,125 +7471,40 @@ STAGE DEPENDENCIES: Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: t1 Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: b (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7610,7 +7525,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7628,7 +7543,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7645,7 +7560,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col3 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7658,7 +7590,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: b (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7671,7 +7620,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7688,7 +7637,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7713,7 +7662,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7740,7 +7706,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7755,7 +7721,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7768,6 +7751,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7848,19 +7848,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: fixob @@ -7877,92 +7877,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7983,7 +7898,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7997,7 +7912,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8014,7 +7929,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8027,7 +7959,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8040,7 +7989,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8057,7 +8006,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8082,7 +8031,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8109,7 +8075,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8124,7 +8090,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8137,6 +8120,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8202,108 +8202,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8316,7 +8231,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8337,7 +8252,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8351,7 +8266,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8368,7 +8283,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8381,7 +8313,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8394,7 +8343,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8411,7 +8360,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8436,110 +8385,10 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col0 (type: int) - outputColumnNames: _col2, _col0 - Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(_col0) - keys: _col2 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 where t1.j=t.j) -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 where t1.j=t.j) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -7 -1 -4 -PREHOOK: query: -- case with empty inner result (t1.j=t.j=NULL) and non-null subquery key(t.i is never null) -explain select t.i from t where t.i NOT IN (select t1.i from t t1 where t1.j=t.j) -PREHOOK: type: QUERY -POSTHOOK: query: -- case with empty inner result (t1.j=t.j=NULL) and non-null subquery key(t.i is never null) -explain select t.i from t where t.i NOT IN (select t1.i from t t1 where t1.j=t.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 4 Map Operator Tree: TableScan - alias: t + alias: t1 Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) @@ -8553,30 +8402,55 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col0 (type: int) + outputColumnNames: _col2, _col0 + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: j (type: int) + aggregations: count(), count(_col0) + keys: _col2 (type: int) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 6 Execution mode: llap - LLAP IO: no inputs - Map 15 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: i (type: int) + keys: j (type: int) mode: hash outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE @@ -8587,7 +8461,20 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 Map Operator Tree: TableScan alias: t1 @@ -8604,28 +8491,56 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 where t1.j=t.j) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 where t1.j=t.j) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +7 +1 +4 +PREHOOK: query: -- case with empty inner result (t1.j=t.j=NULL) and non-null subquery key(t.i is never null) +explain select t.i from t where t.i NOT IN (select t1.i from t t1 where t1.j=t.j) +PREHOOK: type: QUERY +POSTHOOK: query: -- case with empty inner result (t1.j=t.j=NULL) and non-null subquery key(t.i is never null) +explain select t.i from t where t.i NOT IN (select t1.i from t t1 where t1.j=t.j) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8638,7 +8553,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8659,7 +8574,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8673,7 +8588,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8690,7 +8605,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8703,7 +8635,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8716,7 +8665,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8733,7 +8682,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8758,7 +8707,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8785,7 +8751,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8800,7 +8766,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8813,6 +8796,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8829,7 +8829,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t #### A masked pattern was here #### 7 -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- case with non-empty inner result and null subquery key(t.j is null) explain select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY @@ -8845,13 +8845,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -8866,48 +8866,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8924,7 +8883,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8949,19 +8908,60 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8985,7 +8985,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -8996,7 +8996,7 @@ POSTHOOK: Input: default@t #### A masked pattern was here #### 1 4 -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: -- case with non-empty inner result and non-null subquery key(t.i is never null) explain select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY @@ -9012,13 +9012,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -9033,48 +9033,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9091,7 +9050,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9116,7 +9075,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9128,7 +9107,28 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9152,7 +9152,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -9165,8 +9165,8 @@ PREHOOK: query: drop table t1 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: -- corr predicate is not equi explain select * from src b @@ -9194,19 +9194,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -9223,94 +9223,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9338,7 +9251,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 294768 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9356,7 +9269,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9373,7 +9286,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9385,7 +9315,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9398,7 +9345,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9415,7 +9362,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9440,7 +9387,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9470,7 +9435,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9485,7 +9450,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9497,6 +9479,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) + Vertex 9 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -9504,8 +9504,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git a/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out b/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out index 852d919..66c58b9 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out @@ -6,8 +6,8 @@ POSTHOOK: query: CREATE TABLE table_7 (int_col INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@table_7 -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain SELECT (t1.int_col) * (t1.int_col) AS int_col @@ -55,13 +55,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: table_7 @@ -73,50 +73,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(), count(false) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: false (type: boolean), true (type: boolean) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean), _col1 (type: boolean) - sort order: ++ - Map-reduce partition columns: _col0 (type: boolean), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -156,7 +113,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count(), count(false) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -175,7 +153,29 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: false (type: boolean), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: boolean), _col1 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index cf963b2..a798e9e 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -128,31 +128,31 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 32 <- Map 31 (SIMPLE_EDGE) - Reducer 34 <- Map 33 (SIMPLE_EDGE) - Reducer 4 <- Reducer 21 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -173,7 +173,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: a @@ -192,199 +192,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 31 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 33 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -405,7 +213,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -423,7 +231,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -440,7 +248,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -453,7 +280,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -466,7 +312,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -483,7 +350,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -500,7 +367,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -528,7 +395,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -541,7 +408,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -568,7 +454,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -583,7 +469,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -596,7 +501,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -617,7 +541,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -635,7 +559,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -659,7 +583,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -676,7 +600,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 32 + Vertex 31 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 32 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -689,7 +632,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 33 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -702,7 +664,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -720,7 +682,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -747,7 +728,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -762,7 +743,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out b/ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out index e68f60f..c8e8ad8 100644 --- a/ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out @@ -518,7 +518,7 @@ Keys:val 13.0 1 17.0 1 46.0 1 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 4' is a cross product PREHOOK: query: -- join followed by join SELECT * FROM diff --git a/ql/src/test/results/clientpositive/llap/temp_table.q.out b/ql/src/test/results/clientpositive/llap/temp_table.q.out index e97b3dd..a14b118 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -86,7 +86,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -171,10 +171,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -191,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -246,12 +246,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -268,24 +270,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: bar - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -302,8 +287,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 + Vertex 4 + Map Operator Tree: + TableScan + alias: bar + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out b/ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out index c83da8b..f9d36da 100644 --- a/ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out @@ -79,11 +79,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test @@ -103,7 +103,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -112,7 +112,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: some inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: test1 @@ -131,20 +145,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_dml.q.out b/ql/src/test/results/clientpositive/llap/tez_dml.q.out index d09c1f9..b0f44a5 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -20,11 +20,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -47,7 +47,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -61,7 +61,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -452,7 +452,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_src @@ -875,7 +875,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index 25c6f15..b44fdf9 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -49,27 +49,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +65,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -99,6 +79,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -157,11 +157,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -180,26 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -218,7 +199,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -233,6 +214,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -284,12 +284,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -309,26 +309,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -351,7 +332,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -365,7 +346,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -379,6 +360,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -438,11 +438,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -462,27 +462,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -493,7 +473,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -501,7 +481,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -515,6 +495,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -573,11 +573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -596,26 +596,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -625,7 +606,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -637,7 +618,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -652,6 +633,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -703,12 +703,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -728,26 +728,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -758,7 +739,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -773,7 +754,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,7 +768,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -801,6 +782,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out index b44ab7d..e6aeb4e 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out @@ -35,11 +35,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -59,7 +59,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -78,7 +110,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -97,38 +129,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -230,11 +230,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -254,7 +254,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -273,7 +305,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -292,38 +324,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -423,11 +423,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -447,7 +447,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -466,7 +498,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -485,38 +517,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out index 1909c30..beffa5d 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out @@ -19,11 +19,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -42,23 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -74,7 +58,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -98,6 +82,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,23 +148,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -185,7 +169,7 @@ STAGE PLANS: 1 UDFToInteger(_col0) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2 Data size: 620 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 @@ -197,6 +181,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out b/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out index 2859946..56ba40b 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out @@ -35,11 +35,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -58,26 +58,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: orc_src - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -96,7 +77,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -111,6 +92,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: orc_src + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -150,91 +150,35 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 8 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: y - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -244,18 +188,18 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -265,20 +209,20 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 50261 Data size: 8946458 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 50261 Data size: 8946458 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -286,20 +230,76 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out b/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index fd18117..31dd354 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -243,7 +243,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -260,13 +260,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1144,10 +1144,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1227,7 +1227,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1244,13 +1244,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out index e223666..d09219c 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out @@ -17,13 +17,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -40,40 +40,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -89,7 +56,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -102,7 +69,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -118,7 +85,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -132,6 +99,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out index 18b0a5b..7d99e1e 100644 --- a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out @@ -15,13 +15,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -38,40 +38,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -87,7 +54,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,7 +67,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -116,7 +83,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -130,6 +97,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_self_join.q.out b/ql/src/test/results/clientpositive/llap/tez_self_join.q.out index c3a993d..8b0a55d 100644 --- a/ql/src/test/results/clientpositive/llap/tez_self_join.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_self_join.q.out @@ -38,7 +38,7 @@ POSTHOOK: query: insert into table tez_self_join2 values(1),(2),(3) POSTHOOK: type: QUERY POSTHOOK: Output: default@tez_self_join2 POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select s.id2, s.id3 from @@ -70,11 +70,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: self1 @@ -92,41 +92,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: self2 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('ab' = id3) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tez_self_join2 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: id1 is not null (type: boolean) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -147,7 +113,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -169,6 +135,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: self2 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('ab' = id3) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: tez_self_join2 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: id1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -176,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select s.id2, s.id3 from ( diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out b/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out index 8333037..8d880ca 100644 --- a/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -178,7 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -228,7 +228,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -537,10 +537,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -593,7 +593,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -643,10 +643,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -699,7 +699,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -749,10 +749,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -792,7 +792,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -838,10 +838,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -881,7 +881,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out b/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out index 2d6b1a9..91d6645 100644 --- a/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_smb_main.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -36,7 +36,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -56,24 +74,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -255,11 +255,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -278,7 +278,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -310,7 +310,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -366,11 +366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -389,7 +389,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -408,7 +408,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -421,7 +421,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -477,11 +477,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -500,7 +500,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -519,7 +519,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -532,7 +532,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -584,11 +584,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -626,7 +626,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -638,7 +638,22 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -657,21 +672,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -709,11 +709,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -732,26 +732,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -770,7 +751,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -785,6 +766,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -828,13 +828,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -871,7 +873,41 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -890,7 +926,7 @@ STAGE PLANS: Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -909,42 +945,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -965,11 +965,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -988,7 +988,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1007,7 +1007,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1020,7 +1020,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1070,11 +1070,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1094,7 +1094,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1114,7 +1114,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1123,7 +1123,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1136,7 +1136,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1155,21 +1170,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1215,12 +1215,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (CUSTOM_EDGE), Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (CO_PARTITION_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1240,7 +1242,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1249,7 +1251,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1262,7 +1264,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -1281,7 +1298,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s2 @@ -1300,7 +1317,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1313,7 +1330,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: b @@ -1337,23 +1354,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1388,12 +1388,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -1411,7 +1411,19 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -1429,19 +1441,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1455,7 +1455,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1466,7 +1466,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_union.q.out b/ql/src/test/results/clientpositive/llap/tez_union.q.out index 9955e12..8964324 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -17,11 +17,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -41,7 +43,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -52,7 +54,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -71,7 +73,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -89,8 +91,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -171,15 +171,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 7 (CONTAINS) - Map 8 <- Union 7 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 7 (CONTAINS) + Vertex 8 <- Union 7 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -198,7 +202,41 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: src @@ -217,7 +255,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: src @@ -236,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: src @@ -255,44 +293,6 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 7 - Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -351,11 +351,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) - Map 4 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) + Vertex 4 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -379,7 +381,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src @@ -399,7 +401,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -410,7 +412,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src @@ -430,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -441,8 +443,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -529,13 +529,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 10 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 10 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 7 <- Map 10 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 7 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -555,7 +557,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -567,8 +569,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -581,7 +583,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s8 @@ -610,7 +612,21 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -629,7 +645,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s4 @@ -649,7 +665,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -661,8 +677,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -675,7 +691,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: s5 @@ -694,7 +710,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s6 @@ -714,7 +730,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -726,8 +742,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -740,7 +756,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: s7 @@ -759,7 +775,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: s1 @@ -788,22 +804,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -971,11 +971,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -995,7 +997,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1006,7 +1008,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -1025,7 +1027,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s4 @@ -1045,7 +1047,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1056,7 +1058,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s5 @@ -1075,8 +1077,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1164,11 +1164,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 3 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1185,7 +1187,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1196,7 +1198,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1213,7 +1215,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1224,7 +1226,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1247,8 +1249,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1275,12 +1275,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (BROADCAST_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1296,7 +1298,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1312,7 +1314,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1340,8 +1342,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_union2.q.out b/ql/src/test/results/clientpositive/llap/tez_union2.q.out index 51d45d2..e1a4bd0 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union2.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union2.q.out @@ -41,14 +41,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS), Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -69,28 +73,22 @@ STAGE PLANS: Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -111,22 +109,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -157,10 +140,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out b/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 2ca78d7..45f39d3 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -53,11 +53,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: dummy @@ -80,7 +82,7 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dummy @@ -103,8 +105,6 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out b/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out index 22f24c9..b9b061a 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out @@ -147,16 +147,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -182,7 +184,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: v @@ -197,7 +199,50 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (_col1 <= '2014-09-02') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y @@ -223,7 +268,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: z @@ -249,7 +294,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -265,50 +310,7 @@ STAGE PLANS: value expressions: u (type: bigint) Execution mode: llap LLAP IO: unknown - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: (_col1 <= '2014-09-02') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -330,7 +332,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -343,8 +345,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index 113a925..9908492 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -57,14 +57,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -82,73 +84,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s0 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -182,7 +118,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -199,7 +135,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -216,8 +152,72 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: s0 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -956,14 +956,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -996,58 +998,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1064,7 +1015,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1081,7 +1032,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1115,8 +1084,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -1851,14 +1851,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -1891,58 +1893,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1959,7 +1910,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1976,7 +1927,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2010,8 +1979,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -2738,13 +2738,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2762,40 +2764,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2829,7 +2798,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2846,7 +2815,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2863,8 +2832,39 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -3585,13 +3585,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -3609,28 +3611,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3652,7 +3633,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3689,7 +3670,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3706,8 +3687,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out index 7664c44..64bf2eb 100644 --- a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -49,27 +49,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +65,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -99,6 +79,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -157,11 +157,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -180,26 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -218,7 +199,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -233,6 +214,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -284,12 +284,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -309,26 +309,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -351,7 +332,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -365,7 +346,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -379,6 +360,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -438,11 +438,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -462,27 +462,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -493,7 +473,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -501,7 +481,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -515,6 +495,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -573,11 +573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -596,26 +596,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -625,7 +606,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -637,7 +618,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -652,6 +633,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -703,12 +703,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -728,26 +728,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -758,7 +739,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -773,7 +754,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -787,7 +768,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -801,6 +782,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out index 189841a..142274e 100644 --- a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out @@ -35,11 +35,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -59,7 +59,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -78,7 +110,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -97,38 +129,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -230,11 +230,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -254,7 +254,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -273,7 +305,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -292,38 +324,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -423,11 +423,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -447,7 +447,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -466,7 +498,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -485,38 +517,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union2.q.out b/ql/src/test/results/clientpositive/llap/union2.q.out index c1f6837..670b878 100644 --- a/ql/src/test/results/clientpositive/llap/union2.q.out +++ b/ql/src/test/results/clientpositive/llap/union2.q.out @@ -21,12 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -46,7 +48,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -66,23 +83,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union3.q.out b/ql/src/test/results/clientpositive/llap/union3.q.out index 6e28152..7ea4294 100644 --- a/ql/src/test/results/clientpositive/llap/union3.q.out +++ b/ql/src/test/results/clientpositive/llap/union3.q.out @@ -45,62 +45,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -116,7 +70,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Limit @@ -131,7 +85,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Limit @@ -146,7 +100,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -160,7 +114,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Limit @@ -175,7 +145,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Limit @@ -190,8 +176,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union4.q.out b/ql/src/test/results/clientpositive/llap/union4.q.out index 85d60bf..361d0b8 100644 --- a/ql/src/test/results/clientpositive/llap/union4.q.out +++ b/ql/src/test/results/clientpositive/llap/union4.q.out @@ -39,11 +39,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -61,25 +63,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -103,7 +87,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -127,8 +129,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/union5.q.out b/ql/src/test/results/clientpositive/llap/union5.q.out index 856f9f6..c0c9a6c 100644 --- a/ql/src/test/results/clientpositive/llap/union5.q.out +++ b/ql/src/test/results/clientpositive/llap/union5.q.out @@ -23,12 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -48,27 +50,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -92,7 +74,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -108,7 +90,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -132,8 +134,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union6.q.out b/ql/src/test/results/clientpositive/llap/union6.q.out index 1b1edf8..44b924e 100644 --- a/ql/src/test/results/clientpositive/llap/union6.q.out +++ b/ql/src/test/results/clientpositive/llap/union6.q.out @@ -37,11 +37,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 4 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -59,26 +61,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -98,8 +81,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/union7.q.out b/ql/src/test/results/clientpositive/llap/union7.q.out index 4039b7d..ff60fb9 100644 --- a/ql/src/test/results/clientpositive/llap/union7.q.out +++ b/ql/src/test/results/clientpositive/llap/union7.q.out @@ -23,12 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -48,30 +50,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -95,7 +74,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -111,8 +90,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union8.q.out b/ql/src/test/results/clientpositive/llap/union8.q.out index 1e54073..8275585 100644 --- a/ql/src/test/results/clientpositive/llap/union8.q.out +++ b/ql/src/test/results/clientpositive/llap/union8.q.out @@ -23,12 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -46,7 +48,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -64,7 +66,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -82,8 +84,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union9.q.out b/ql/src/test/results/clientpositive/llap/union9.q.out index bb70d74..2d8639d 100644 --- a/ql/src/test/results/clientpositive/llap/union9.q.out +++ b/ql/src/test/results/clientpositive/llap/union9.q.out @@ -23,13 +23,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -49,7 +51,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -69,7 +86,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s3 @@ -89,23 +106,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out index 797b947..0de9e0b 100644 --- a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out +++ b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out @@ -145,12 +145,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -170,7 +170,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: inputtbl2 @@ -190,7 +205,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: inputtbl3 @@ -210,37 +240,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -367,12 +367,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -395,7 +395,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: int), _col3 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: inputtbl2 @@ -418,7 +434,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: int), _col3 (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: inputtbl3 @@ -441,39 +473,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: int), _col3 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -542,12 +542,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -567,7 +567,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: inputtbl2 @@ -587,7 +602,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: inputtbl3 @@ -607,37 +637,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -705,12 +705,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -730,7 +732,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: inputtbl2 @@ -750,7 +767,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: inputtbl3 @@ -770,22 +802,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -800,23 +817,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/union_stats.q.out b/ql/src/test/results/clientpositive/llap/union_stats.q.out index d42fca4..0de8786 100644 --- a/ql/src/test/results/clientpositive/llap/union_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -14,11 +14,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +102,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -182,8 +184,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/union_top_level.q.out b/ql/src/test/results/clientpositive/llap/union_top_level.q.out index 6ea1f7e..a9b2220 100644 --- a/ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ b/ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -27,12 +27,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -54,7 +56,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -76,7 +99,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: src @@ -98,49 +142,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -161,8 +163,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -214,52 +214,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -279,26 +242,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -321,7 +265,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -338,7 +282,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -361,7 +344,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -378,8 +361,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + Vertex 9 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -449,12 +449,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -476,51 +478,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -542,7 +500,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -564,7 +544,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -586,8 +588,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -684,12 +684,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -711,51 +713,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -777,7 +735,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -799,7 +779,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -821,8 +823,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -904,12 +904,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -931,51 +933,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -997,7 +955,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1019,7 +999,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1041,8 +1043,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out index 4a636e3..7d4e789 100644 --- a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out @@ -121,7 +121,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -189,7 +189,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -257,7 +257,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -325,7 +325,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -393,7 +393,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -461,7 +461,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -519,7 +519,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -613,7 +613,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -671,7 +671,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -765,7 +765,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -825,10 +825,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -851,7 +851,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -902,10 +902,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -928,7 +928,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out b/ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out index d4a9747..239f8e1 100644 --- a/ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -139,7 +139,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out index 420d17a..7a05417 100644 --- a/ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out @@ -38,26 +38,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_14] - Group By Operator [GBY_13] (rows=1 width=188) - Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=188) - Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"] - Select Operator [SEL_10] (rows=3 width=102) - Output:["dt","greg_dt"] - Filter Operator [FIL_9] (rows=3 width=102) - predicate:(id = 5) - TableScan [TS_0] (rows=7 width=102) - default@testvec,testvec,Tbl:COMPLETE,Col:NONE,Output:["id","dt","greg_dt"] + Vertex 2 PREHOOK: query: select max(dt), max(greg_dt) from testvec where id=5 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index 00b1aa7..3b19971 100644 --- a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -57,34 +57,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -126,43 +105,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 vectorized, llap - File Output Operator [FS_31] - Group By Operator [GBY_30] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_29] - Group By Operator [GBY_28] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_27] (rows=5 width=93) - Group By Operator [GBY_26] (rows=5 width=93) - Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=11 width=93) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_24] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_23] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_22] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: select count(*) from ( @@ -230,68 +180,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_32] - Select Operator [SEL_31] (rows=5 width=102) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_49] (rows=5 width=102) - Conds:RS_51._col0=RS_53._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 2 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_45] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_42] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_41] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Reducer 6 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_53] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_47] (rows=11 width=93) - Conds:SEL_16._col0=SEL_19._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_19] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_44] (rows=10 width=93) - predicate:key is not null - TableScan [TS_17] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_16] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_43] (rows=10 width=93) - predicate:key is not null - TableScan [TS_14] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: select src1.key, src1.cnt1, src2.cnt1 from ( @@ -358,34 +255,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select a.key as key, a.value as value from tbl1 a where key < 6) subq1 @@ -437,34 +313,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from ( @@ -540,34 +395,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_3] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from ( @@ -633,34 +467,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=3 width=93) - predicate:(key < 8) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=3 width=93) - predicate:(key < 8) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1 @@ -702,41 +515,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 vectorized, llap - File Output Operator [FS_29] - Group By Operator [GBY_28] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_21] (rows=11 width=93) - Conds:RS_24._col0=RS_27._col0(Inner) - <-Map 1 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_22] (rows=10 width=93) - predicate:(key + 1) is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_25] (rows=10 width=93) - predicate:(key + 1) is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: select count(*) from (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1 @@ -774,34 +560,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select a.key as key, a.value as value from tbl1 a where key < 6) subq1 @@ -845,40 +610,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_32] - Group By Operator [GBY_31] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - Group By Operator [GBY_14] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_28] (rows=6 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner),SEL_2._col0=SEL_8._col0(Inner) - <-Select Operator [SEL_5] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_26] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_8] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_27] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_6] (rows=10 width=93) - default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=3 width=93) - Output:["_col0"] - Filter Operator [FIL_25] (rows=3 width=93) - predicate:(key < 6) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select a.key as key, a.value as value from tbl1 a where key < 6) subq1 @@ -938,34 +676,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_22] - Group By Operator [GBY_21] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) - Conds:SEL_2._col0=SEL_5._col0(Inner) - <-Select Operator [SEL_5] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=1 width=93) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1 width=93) - predicate:((key < 8) and (key < 6)) - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from ( select subq2.key as key, subq2.value as value1, b.value as value2 from @@ -1042,30 +759,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Map 1 llap - File Output Operator [FS_11] - table:{"name:":"default.dest1"} - Select Operator [SEL_10] (rows=11 width=93) - Output:["_col0","_col1"] - Select Operator [SEL_9] (rows=11 width=93) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_18] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1","_col3"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - File Output Operator [FS_13] - table:{"name:":"default.dest2"} - Please refer to the previous Select Operator [SEL_9] + Vertex 1 Stage-5 Stats-Aggr Operator Stage-1 @@ -1195,7 +889,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -1205,37 +899,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 2 vectorized, llap - File Output Operator [FS_27] - table:{"name:":"default.dest2"} - Select Operator [SEL_26] (rows=5 width=93) - Output:["_col0","_col1"] - Group By Operator [GBY_25] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - File Output Operator [FS_11] - table:{"name:":"default.dest1"} - Merge Join Operator [MERGEJOIN_23] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_22] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_21] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_12] (rows=11 width=93) - Output:["_col0"] - Please refer to the previous Merge Join Operator [MERGEJOIN_23] + Vertex 2 Stage-5 Stats-Aggr Operator Stage-1 diff --git a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out index c62a415..1d20aa0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out @@ -68,7 +68,7 @@ POSTHOOK: Output: default@TINT POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ] tint_txt.rnum tint_txt.cint -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -105,7 +105,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col1 (type: int), _col3 (type: smallint), CASE WHEN (_col1 BETWEEN _col3 AND _col3) THEN ('Ok') ELSE ('NoOk') END (type: string) @@ -120,7 +120,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -142,7 +142,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY PREHOOK: Input: default@tint @@ -179,7 +179,7 @@ tint.rnum tsint.rnum tint.cint tsint.csint between_col 4 2 10 0 NoOk 4 3 10 1 NoOk 4 4 10 10 Ok -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY @@ -196,10 +196,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -216,7 +216,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col1 BETWEEN _col3 AND _col3 (type: boolean) @@ -234,7 +234,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -256,7 +256,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY PREHOOK: Input: default@tint diff --git a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out index 6a523f1..9bc6910 100644 --- a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -46,7 +46,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -80,10 +80,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -139,10 +139,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -160,7 +160,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -194,10 +194,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -218,7 +218,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -274,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -329,7 +329,7 @@ STAGE PLANS: Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -384,7 +384,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -418,10 +418,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -442,7 +442,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -726,11 +726,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -753,7 +753,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -767,7 +767,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -801,11 +801,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -828,7 +828,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -842,7 +842,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -876,11 +876,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -903,7 +903,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -917,7 +917,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -951,11 +951,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -978,7 +978,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -992,7 +992,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out b/ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out index a510e38..efc1b66 100644 --- a/ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out @@ -114,11 +114,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -138,7 +138,7 @@ STAGE PLANS: 1 _col10 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21) (type: int) @@ -155,7 +155,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -175,21 +190,6 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)) Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -227,10 +227,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: hundredorc @@ -253,7 +253,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -339,10 +339,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -362,7 +362,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: binary), _col3 (type: binary) @@ -377,7 +377,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git a/ql/src/test/results/clientpositive/llap/vector_bucket.q.out b/ql/src/test/results/clientpositive/llap/vector_bucket.q.out index b475999..05354f6 100644 --- a/ql/src/test/results/clientpositive/llap/vector_bucket.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_bucket.q.out @@ -23,10 +23,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: values__tmp__table__1 @@ -42,7 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out index cd67e7e..9b3d8b5 100644 --- a/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out @@ -120,11 +120,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -148,7 +148,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -163,7 +163,7 @@ STAGE PLANS: Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_char_2.q.out b/ql/src/test/results/clientpositive/llap/vector_char_2.q.out index b7b2ba5..80e666c 100644 --- a/ql/src/test/results/clientpositive/llap/vector_char_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_char_2.q.out @@ -68,11 +68,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -96,7 +96,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -200,11 +200,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -228,7 +228,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -243,7 +243,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_char_4.q.out b/ql/src/test/results/clientpositive/llap/vector_char_4.q.out index 6d55ab0..cedcd8a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_char_4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_char_4.q.out @@ -138,7 +138,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc diff --git a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out index 1af8b3d..e0e1f2a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out @@ -139,11 +139,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -163,7 +163,7 @@ STAGE PLANS: 1 _col1 (type: char(10)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -172,7 +172,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -192,20 +206,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -245,11 +245,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -269,7 +269,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -289,7 +289,7 @@ STAGE PLANS: 1 _col1 (type: char(20)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -298,7 +298,7 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -353,11 +353,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -377,7 +377,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -386,7 +386,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -406,20 +420,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out index 7c35009..777837e 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out @@ -35,10 +35,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -61,7 +61,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,7 +121,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -267,7 +267,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out index 04ec41a..9c4e862 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: m @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), CASE WHEN ((COALESCE(_col2,5) > 1)) THEN (_col2) ELSE (null) END (type: bigint) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: n diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index 565f9ab..974d342 100644 --- a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -150,7 +150,7 @@ b str two line1 four line2 six line3 -Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT orc_create_staging.*, src1.key FROM orc_create_staging cross join src src1 cross join orc_create_staging spam1 cross join orc_create_staging spam2 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out index d271f04..9777354 100644 --- a/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out @@ -37,10 +37,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -60,7 +60,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 10090 Data size: 2540021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -71,7 +71,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2b @@ -179,7 +179,7 @@ STAGE PLANS: 1 a[1] (type: int) outputColumnNames: _col0, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col4 (type: array) @@ -194,7 +194,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2a diff --git a/ql/src/test/results/clientpositive/llap/vector_count.q.out b/ql/src/test/results/clientpositive/llap/vector_count.q.out index 3b9d9f9..8ecaca9 100644 --- a/ql/src/test/results/clientpositive/llap/vector_count.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_count.q.out @@ -60,10 +60,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -86,7 +86,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -134,10 +134,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -159,7 +159,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -203,10 +203,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -223,7 +223,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -271,10 +271,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -289,7 +289,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out index 3d67664..c95cfc3 100644 --- a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out @@ -1248,11 +1248,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: web_sales @@ -1273,7 +1273,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 3520000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1290,7 +1290,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_data_types.q.out b/ql/src/test/results/clientpositive/llap/vector_data_types.q.out index a7a74c3..1da22da 100644 --- a/ql/src/test/results/clientpositive/llap/vector_data_types.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_data_types.q.out @@ -110,10 +110,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -130,7 +130,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -206,10 +206,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -226,7 +226,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_date_1.q.out b/ql/src/test/results/clientpositive/llap/vector_date_1.q.out index 64d5be7..28c360f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_date_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_date_1.q.out @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -195,10 +195,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -307,10 +307,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -326,7 +326,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -417,10 +417,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -436,7 +436,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -529,10 +529,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -551,7 +551,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -645,10 +645,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -666,7 +666,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out index b3ff557..e8d5454 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -65,7 +65,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -110,10 +110,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -128,7 +128,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -173,10 +173,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -191,7 +191,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -254,7 +254,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -299,10 +299,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -317,7 +317,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -380,7 +380,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -425,10 +425,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -443,7 +443,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -488,10 +488,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out index 5576078..4540f14 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out @@ -48,10 +48,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal @@ -66,7 +66,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out index db5e183..0359887 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out @@ -36,10 +36,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -54,7 +54,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -99,10 +99,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -117,7 +117,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -162,10 +162,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -180,7 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -225,10 +225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -243,7 +243,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -306,7 +306,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -414,10 +414,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -432,7 +432,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -495,7 +495,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -614,10 +614,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -632,7 +632,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -677,10 +677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -695,7 +695,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -740,10 +740,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -758,7 +758,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -803,10 +803,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -821,7 +821,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -866,10 +866,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -884,7 +884,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -929,10 +929,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -947,7 +947,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -992,10 +992,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1010,7 +1010,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1055,10 +1055,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1070,7 +1070,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1115,10 +1115,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1130,7 +1130,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1175,10 +1175,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1190,7 +1190,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1235,10 +1235,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1250,7 +1250,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1286,10 +1286,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1301,7 +1301,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1346,10 +1346,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1361,7 +1361,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1406,10 +1406,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1421,7 +1421,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1466,10 +1466,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1481,7 +1481,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1526,10 +1526,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1541,7 +1541,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1586,10 +1586,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1601,7 +1601,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1646,10 +1646,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1661,7 +1661,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out index 6faf453..11e04d3 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out @@ -49,10 +49,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -75,7 +75,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -156,10 +156,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -182,7 +182,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out index 361c46b..808dacd 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out @@ -87,10 +87,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -110,7 +110,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -121,7 +121,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out index 8ccc8cf..4a98c47 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out @@ -558,10 +558,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_precision @@ -581,7 +581,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: decimal(30,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out index 8e7cd63..d3b65c2 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -62,7 +62,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -107,10 +107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -126,7 +126,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -197,10 +197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -261,10 +261,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -280,7 +280,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -415,10 +415,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -434,7 +434,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out index c653e5c..316e343 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out @@ -65,10 +65,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_1_orc @@ -84,7 +84,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -178,10 +178,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_2_orc @@ -197,7 +197,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(25,4)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(22,1)), _col13 (type: decimal(23,2)), _col14 (type: decimal(24,3)), _col15 (type: decimal(25,4)), _col16 (type: decimal(21,0)), _col17 (type: decimal(21,0)), _col18 (type: decimal(21,0)), _col19 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -318,10 +318,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_3_orc @@ -337,7 +337,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(21,0)), _col3 (type: decimal(21,0)), _col4 (type: decimal(21,0)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)), _col13 (type: decimal(21,0)), _col14 (type: decimal(21,0)), _col15 (type: decimal(21,0)), _col16 (type: decimal(21,0)), _col17 (type: decimal(22,1)), _col18 (type: decimal(23,2)), _col19 (type: decimal(24,3)), _col20 (type: decimal(25,4)), _col21 (type: decimal(26,5)), _col22 (type: decimal(27,6)), _col23 (type: decimal(28,7)), _col24 (type: decimal(29,8)), _col25 (type: decimal(30,9)), _col26 (type: decimal(31,10)), _col27 (type: decimal(32,11)), _col28 (type: decimal(33,12)), _col29 (type: decimal(34,13)), _col31 (type: decimal(35,14)), _col32 (type: decimal(36,15)), _col33 (type: decimal(37,16)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -447,10 +447,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_4_orc @@ -466,7 +466,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,9)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out index 87fde0e..c48cef8 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out @@ -63,7 +63,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -147,7 +147,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -231,7 +231,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -315,7 +315,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -401,7 +401,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -485,7 +485,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -569,7 +569,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -653,7 +653,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -739,7 +739,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -823,7 +823,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -895,7 +895,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -979,7 +979,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1063,7 +1063,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1149,7 +1149,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1199,7 +1199,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1249,7 +1249,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1332,7 +1332,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1405,7 +1405,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1478,7 +1478,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1564,7 +1564,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1650,11 +1650,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1677,7 +1677,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,10)), _col2 (type: bigint), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1695,7 +1695,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(38,18)), _col2 (type: decimal(24,14)), _col3 (type: decimal(30,10)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1756,7 +1756,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1909,7 +1909,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1995,7 +1995,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2081,7 +2081,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2167,7 +2167,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2253,7 +2253,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2339,10 +2339,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2365,7 +2365,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2428,10 +2428,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2454,7 +2454,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2517,10 +2517,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2540,7 +2540,7 @@ STAGE PLANS: value expressions: _col0 (type: array) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2586,10 +2586,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2609,7 +2609,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2655,10 +2655,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2678,7 +2678,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2724,10 +2724,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2747,7 +2747,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out index 5ea9f4d..3fda0f9 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 @@ -127,7 +127,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 diff --git a/ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out index 188c624..1eeacda 100644 --- a/ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out @@ -120,10 +120,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -144,7 +144,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby4.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby4.q.out index 1e24e81..c7bfec8 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby4.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -61,7 +61,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -74,7 +74,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby6.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby6.q.out index 758f70c..93cfc85 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby6.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby6.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -61,7 +61,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -74,7 +74,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out index fb5dfe6..6dc1c9f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out @@ -120,10 +120,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out index 3ab6eb9..d6b7ca7 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: -- HIVE-12738 -- We are checking if a MapJoin after a GroupBy will work properly. explain select * @@ -18,61 +18,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Reducer 4 (BROADCAST_EDGE), Reducer 6 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - Select Operator [SEL_22] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_21] (rows=500 width=198) - predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) - Map Join Operator [MAPJOIN_29] (rows=500 width=198) - Conds:MAPJOIN_28._col0=RS_35._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 6 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_35] - PartitionCols:_col0 - Group By Operator [GBY_34] (rows=205 width=91) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_11] (rows=205 width=91) - Output:["_col0","_col1"],keys:_col0, true - Select Operator [SEL_9] (rows=500 width=87) - Output:["_col0"] - TableScan [TS_8] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map Join Operator [MAPJOIN_28] (rows=500 width=194) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_33] - Group By Operator [GBY_32] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(key)"] - Select Operator [SEL_3] (rows=500 width=87) - Output:["key"] - TableScan [TS_2] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select * from src where not key in diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index c4bcbab..8c295c4 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -261,7 +261,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -382,7 +382,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -404,7 +404,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -566,11 +566,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -593,7 +593,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -617,7 +617,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -779,11 +779,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -806,7 +806,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -830,7 +830,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out index 2fb434c..2e23e98 100644 --- a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out @@ -146,10 +146,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -170,7 +170,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,10 +231,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -255,7 +255,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -319,10 +319,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -343,7 +343,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out b/ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out index e939c67..d6dafcc 100644 --- a/ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out @@ -180,7 +180,7 @@ POSTHOOK: Lineage: customer_demographics.cd_education_status SIMPLE [(customer_d POSTHOOK: Lineage: customer_demographics.cd_gender SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_gender, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_marital_status SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_marital_status, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_purchase_estimate SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_purchase_estimate, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: explain select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or @@ -200,11 +200,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: customer_demographics @@ -215,7 +215,7 @@ STAGE PLANS: value expressions: cd_demo_sk (type: int), cd_marital_status (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: store_sales @@ -228,7 +228,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 200000 Data size: 92055200 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col0 = _col16) and (_col2 = 'M')) or ((_col0 = _col16) and (_col2 = 'U'))) (type: boolean) @@ -246,7 +246,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -268,7 +268,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or (customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'U')) diff --git a/ql/src/test/results/clientpositive/llap/vector_inner_join.q.out b/ql/src/test/results/clientpositive/llap/vector_inner_join.q.out index d50123d..dbc9e6d 100644 --- a/ql/src/test/results/clientpositive/llap/vector_inner_join.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_inner_join.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -133,10 +133,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -156,7 +156,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -167,7 +167,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -256,10 +256,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -279,7 +279,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: int) @@ -294,7 +294,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -347,10 +347,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -390,7 +390,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -434,10 +434,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -457,7 +457,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), (_col3 * 2) (type: int), (_col0 * 5) (type: int), _col1 (type: string) @@ -472,7 +472,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -525,10 +525,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -548,7 +548,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col0 (type: int) @@ -563,7 +563,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -616,10 +616,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -639,7 +639,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col2 (type: string), _col1 (type: string) @@ -654,7 +654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -707,10 +707,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -730,7 +730,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col2 (type: int) @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -821,7 +821,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -841,7 +841,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: string) diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out index 995c229..c099ad4 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out @@ -63,10 +63,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -82,7 +82,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -154,10 +154,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -173,7 +173,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -251,10 +251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -270,7 +270,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -381,7 +381,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -485,10 +485,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -504,7 +504,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -590,10 +590,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -609,7 +609,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -677,10 +677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -696,7 +696,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -770,10 +770,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -789,7 +789,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out index 23a977e..ca56ca3 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_2.q.out @@ -119,10 +119,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -138,7 +138,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -307,7 +307,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -457,10 +457,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -476,7 +476,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -626,10 +626,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -645,7 +645,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -783,10 +783,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -804,7 +804,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -937,10 +937,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -958,7 +958,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1081,10 +1081,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1102,7 +1102,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1225,10 +1225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1246,7 +1246,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1381,10 +1381,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1402,7 +1402,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1535,10 +1535,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1556,7 +1556,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out index 464fd2a..b9848c0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out @@ -71,10 +71,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -90,7 +90,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -216,10 +216,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -235,7 +235,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -361,10 +361,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -380,7 +380,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -504,10 +504,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -521,7 +521,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -602,10 +602,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -621,7 +621,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -749,10 +749,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -768,7 +768,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -896,10 +896,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -915,7 +915,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1037,7 +1037,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out index 0bc0e4c..2d5d8a1 100644 --- a/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab_a_1korc @@ -212,7 +212,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: interval_day_time) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1100 Data size: 506290 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time) @@ -227,7 +227,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: vectortab_b_1korc diff --git a/ql/src/test/results/clientpositive/llap/vector_join30.q.out b/ql/src/test/results/clientpositive/llap/vector_join30.q.out index bb6916b..1c6902b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_join30.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_join30.q.out @@ -39,12 +39,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -62,26 +62,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -96,7 +77,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -107,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -122,7 +103,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -186,12 +186,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -206,23 +206,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -237,7 +221,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -248,7 +232,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -263,7 +247,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -327,12 +327,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -347,7 +347,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: orcsrc @@ -363,19 +375,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -390,7 +390,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -401,7 +401,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -473,13 +473,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -497,45 +497,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -552,8 +514,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 - 2 Reducer 7 + 1 Vertex 5 + 2 Vertex 7 Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -564,7 +526,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -579,7 +541,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -592,7 +573,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -667,14 +667,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -689,39 +689,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -733,7 +701,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -755,7 +723,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -770,7 +738,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -783,7 +767,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -858,14 +858,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -880,39 +880,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -924,7 +892,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -946,7 +914,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -961,7 +929,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -974,7 +958,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1049,14 +1049,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -1071,39 +1071,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1115,7 +1083,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1137,7 +1105,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1152,7 +1120,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1165,7 +1149,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1240,14 +1240,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -1262,39 +1262,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1306,7 +1274,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1328,7 +1296,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1343,7 +1311,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1356,7 +1340,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_join_filters.q.out b/ql/src/test/results/clientpositive/llap/vector_join_filters.q.out index a010722..0af72d8 100644 --- a/ql/src/test/results/clientpositive/llap/vector_join_filters.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_join_filters.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -40,7 +40,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -50,7 +50,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out b/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out index 95b35b6..2e42e0a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -40,7 +40,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -50,7 +50,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git a/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out b/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out index b9ffa34..840fd4f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -46,7 +46,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 104600 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 3418417 Data size: 27347336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -68,7 +68,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -84,7 +99,7 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -100,21 +115,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out b/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out index 1163d24..3a87888 100644 --- a/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out @@ -95,10 +95,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -118,7 +118,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -133,7 +133,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -185,10 +185,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -208,7 +208,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -223,7 +223,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -275,10 +275,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -298,7 +298,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -313,7 +313,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -365,10 +365,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -388,7 +388,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -403,7 +403,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -455,10 +455,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -478,7 +478,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -493,7 +493,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -545,10 +545,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -568,7 +568,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -583,7 +583,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 diff --git a/ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out b/ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out index b6a3b9a..3d6ad9b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out @@ -247,13 +247,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -273,46 +273,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: unknown - Map 6 - Map Operator Tree: - TableScan - alias: store - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: s_store_sk (type: int), s_state (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: date_dim - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: d_date_sk is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -329,7 +290,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -357,7 +318,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -372,7 +333,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -389,6 +350,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: store + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: s_store_sk (type: int), s_state (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: date_dim + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: d_date_sk is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: d_date_sk (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out index d004d81..cf3d4d1 100644 --- a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -98,12 +98,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -124,7 +126,26 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: void) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -145,27 +166,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: void) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out b/ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out index 92a4cf7..d77d596 100644 --- a/ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out @@ -124,10 +124,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc @@ -143,7 +143,7 @@ STAGE PLANS: value expressions: _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -203,10 +203,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc @@ -222,7 +222,7 @@ STAGE PLANS: value expressions: _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out index 503cf5b..d9abd62 100644 --- a/ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out @@ -116,34 +116,34 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: bo (type: boolean), b (type: bigint) outputColumnNames: bo, b - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(b) keys: bo (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col0 (type: boolean) sort order: + Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -151,22 +151,22 @@ STAGE PLANS: keys: KEY._col0 (type: boolean) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col0 (type: boolean) sort order: - - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: boolean), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -186,6 +186,3 @@ POSTHOOK: query: select bo, max(b) from vectortab2korc group by bo order by bo d POSTHOOK: type: QUERY POSTHOOK: Input: default@vectortab2korc #### A masked pattern was here #### -true 9211455920344088576 -false 9209153648361848832 -NULL 9180098147855769600 diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out index 563d10a..0cb3d9d 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -93,7 +93,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -104,7 +104,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -163,10 +163,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -183,7 +183,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -200,7 +200,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out index cda039f..9921ede 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out @@ -237,10 +237,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -257,7 +257,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 32 Data size: 19648 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -268,7 +268,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -350,10 +350,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -370,7 +370,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 112 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -381,7 +381,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -554,11 +554,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -575,7 +575,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -585,7 +585,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 240 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col0) @@ -598,7 +598,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -614,7 +629,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -630,21 +645,6 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out index 051911b..716de0b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out @@ -253,11 +253,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -274,7 +274,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 57 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -284,7 +284,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 162 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col1) @@ -297,7 +297,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -313,7 +328,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -329,21 +344,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index 5729237..7c86c2b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -253,11 +253,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -274,7 +274,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -283,7 +283,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 80 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -296,7 +296,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -312,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -328,21 +343,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -402,11 +402,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -423,7 +423,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 21 Data size: 1869 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -432,7 +432,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -445,7 +445,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -461,7 +476,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -477,21 +492,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -551,11 +551,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -572,7 +572,7 @@ STAGE PLANS: 1 _col0 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 20 Data size: 1860 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -581,7 +581,7 @@ STAGE PLANS: 0 _col0 (type: int), _col2 (type: string) 1 _col0 (type: int), _col1 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -594,7 +594,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -610,7 +625,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 2120 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -626,21 +641,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1023 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out index 75d783f..54990a5 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out @@ -267,10 +267,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -287,7 +287,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 81 Data size: 49734 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -298,7 +298,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -415,10 +415,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -435,7 +435,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 450 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -446,7 +446,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -924,11 +924,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -945,7 +945,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 81 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -954,7 +954,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1215 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -967,7 +967,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -983,7 +998,7 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -999,21 +1014,6 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out index 8a18738..9109338 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out @@ -89,11 +89,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -109,7 +109,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -122,7 +122,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -138,21 +153,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -202,11 +202,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -225,7 +225,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -238,7 +238,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -254,21 +269,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -318,11 +318,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -341,7 +341,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -354,7 +354,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -370,21 +385,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -434,11 +434,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -457,7 +457,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -470,7 +470,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -486,21 +501,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -554,12 +554,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -576,7 +576,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint) @@ -585,39 +585,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -636,7 +604,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -651,6 +619,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -766,11 +766,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -786,7 +786,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -799,7 +799,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -815,21 +830,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -879,11 +879,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -902,7 +902,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -915,7 +915,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -931,21 +946,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -995,11 +995,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -1018,7 +1018,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1031,7 +1031,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1047,21 +1062,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1111,11 +1111,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -1134,7 +1134,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1147,7 +1147,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1163,21 +1178,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1231,12 +1231,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -1253,7 +1253,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) @@ -1262,39 +1262,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cmodtinyint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1313,7 +1281,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1328,6 +1296,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out index 9369661..a6e411a 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out @@ -143,10 +143,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -163,7 +163,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -177,7 +177,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) @@ -192,7 +192,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -209,7 +209,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 @@ -268,10 +268,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -288,7 +288,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -302,7 +302,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -313,7 +313,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -330,7 +330,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 diff --git a/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out b/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out index ebe895f..dfda509 100644 --- a/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out @@ -91,10 +91,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inventory_part_0 @@ -114,7 +114,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -219,10 +219,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inventory_part_1 @@ -242,7 +242,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -349,10 +349,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inventory_part_2a @@ -372,7 +372,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -460,10 +460,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inventory_part_2b @@ -483,7 +483,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -577,10 +577,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inventory_part_3 @@ -600,7 +600,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/perf/query12.q.out b/ql/src/test/results/clientpositive/perf/query12.q.out index 3f25f56..1712b4f 100644 --- a/ql/src/test/results/clientpositive/perf/query12.q.out +++ b/ql/src/test/results/clientpositive/perf/query12.q.out @@ -3,70 +3,15 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select i_item_desc ,i_category ,i_class ,i_current_price ,i_item_id ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where web_sales.ws_item_sk = item.i_item_sk and item.i_category in ('Jewelry', 'Sports', 'Books') and web_sales.ws_sold_date_sk = date_dim.d_date_sk and date_dim.d_date between '2001-01-12' and '2001-02-11' group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_26] - Limit [LIM_25] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_24] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Select Operator [SEL_21] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_20] (rows=21780404 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_19] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Group By Operator [GBY_17] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_15] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col23)"],keys:_col38, _col41, _col49, _col47, _col42 - Select Operator [SEL_14] (rows=43560808 width=135) - Output:["_col38","_col41","_col49","_col47","_col42","_col23"] - Filter Operator [FIL_32] (rows=43560808 width=135) - predicate:((_col3 = _col37) and (_col0 = _col62)) - Merge Join Operator [MERGEJOIN_37] (rows=174243235 width=135) - Conds:RS_9._col0=RS_11.d_date_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49","_col62"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:d_date_sk - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date_sk is not null and d_date BETWEEN '2001-01-12' AND '2001-02-11') - TableScan [TS_2] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_36] (rows=158402938 width=135) - Conds:RS_4.ws_item_sk=RS_6.i_item_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:ws_item_sk - Filter Operator [FIL_33] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:i_item_sk - Filter Operator [FIL_34] (rows=231000 width=1436) - predicate:(i_item_sk is not null and (i_category) IN ('Jewelry', 'Sports', 'Books')) - TableScan [TS_1] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query13.q.out b/ql/src/test/results/clientpositive/perf/query13.q.out index 2f84cce..7aaf6ae 100644 --- a/ql/src/test/results/clientpositive/perf/query13.q.out +++ b/ql/src/test/results/clientpositive/perf/query13.q.out @@ -109,109 +109,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_43] - Group By Operator [GBY_41] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","sum(VALUE._col3)"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - Group By Operator [GBY_39] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col7)","avg(_col9)","avg(_col10)","sum(_col10)"] - Merge Join Operator [MERGEJOIN_74] (rows=2016666 width=1014) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col7","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_64] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=1833333 width=1014) - Output:["_col0","_col5","_col7","_col8"] - Filter Operator [FIL_33] (rows=1833333 width=1014) - predicate:(((_col17) IN ('KY', 'GA', 'NM') and _col9 BETWEEN 100 AND 200) or ((_col17) IN ('MT', 'OR', 'IN') and _col9 BETWEEN 150 AND 300) or ((_col17) IN ('WI', 'MO', 'WV') and _col9 BETWEEN 50 AND 250)) - Merge Join Operator [MERGEJOIN_73] (rows=11000000 width=1014) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col0","_col5","_col7","_col8","_col9","_col17"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_29] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_27] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_26] (rows=3549306 width=88) - Output:["_col0","_col3","_col5","_col7","_col8","_col9"] - Filter Operator [FIL_25] (rows=3549306 width=88) - predicate:(((_col12 = 'M') and (_col13 = '4 yr Degree') and _col6 BETWEEN 100 AND 150 and (_col15 = 3)) or ((_col12 = 'D') and (_col13 = 'Primary') and _col6 BETWEEN 50 AND 100 and (_col15 = 1)) or ((_col12 = 'U') and (_col13 = 'Advanced Degree') and _col6 BETWEEN 150 AND 200 and (_col15 = 1))) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col3","_col5","_col6","_col7","_col8","_col9","_col12","_col13","_col15"] - Merge Join Operator [MERGEJOIN_72] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col6","_col7","_col8","_col9","_col10","_col12","_col14","_col15"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_68] (rows=1861800 width=385) - predicate:(((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and ((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_71] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_67] (rows=7200 width=107) - predicate:(((hd_dep_count = 3) or (hd_dep_count = 1)) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_70] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col4(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9","_col10"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_65] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col4 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_66] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 100 AND 200 or ss_net_profit BETWEEN 150 AND 300 or ss_net_profit BETWEEN 50 AND 250) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_ext_sales_price","ss_ext_wholesale_cost","ss_net_profit"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query15.q.out b/ql/src/test/results/clientpositive/perf/query15.q.out index 4f4dcc5..53e9ce7 100644 --- a/ql/src/test/results/clientpositive/perf/query15.q.out +++ b/ql/src/test/results/clientpositive/perf/query15.q.out @@ -5,83 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_29] (rows=174233858 width=135) - Output:["_col0","_col1"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Group By Operator [GBY_26] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_23] (rows=348467716 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_47] (rows=348467716 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=316788826 width=135) - Output:["_col0","_col2","_col7"] - Filter Operator [FIL_15] (rows=316788826 width=135) - predicate:((substr(_col4, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792') or (_col3) IN ('CA', 'WA', 'GA') or (_col7 > 500)) - Merge Join Operator [MERGEJOIN_46] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col1(Inner),Output:["_col3","_col4","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_6] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_45] (rows=88000001 width=860) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_zip"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query16.q.out b/ql/src/test/results/clientpositive/perf/query16.q.out index 1f33e43..69e8344 100644 --- a/ql/src/test/results/clientpositive/perf/query16.q.out +++ b/ql/src/test/results/clientpositive/perf/query16.q.out @@ -59,272 +59,29 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 24 <- Map 23 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 25 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_110] - Limit [LIM_109] (rows=1 width=344) - Number of rows:100 - Select Operator [SEL_108] (rows=1 width=344) - Output:["_col0","_col1","_col2"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_107] - Group By Operator [GBY_105] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_104] - Group By Operator [GBY_103] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_102] (rows=231905279 width=135) - Output:["_col4","_col5","_col6"] - Filter Operator [FIL_101] (rows=231905279 width=135) - predicate:_col16 is null - Select Operator [SEL_100] (rows=463810558 width=135) - Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_193] (rows=463810558 width=135) - Conds:RS_97._col4, _col3=RS_98._col1, _col0(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1, _col0 - Select Operator [SEL_84] (rows=158394413 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_83] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1 - Group By Operator [GBY_81] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col3, _col2 - Select Operator [SEL_80] (rows=316788826 width=135) - Output:["_col3","_col2"] - Filter Operator [FIL_79] (rows=316788826 width=135) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_188] (rows=316788826 width=135) - Conds:RS_76._col1=RS_77._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col1 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_48] (rows=287989836 width=135) - default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col1 - Select Operator [SEL_75] (rows=191657247 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_74] (rows=191657247 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0, _col1 - Group By Operator [GBY_72] (rows=383314495 width=135) - Output:["_col0","_col1"],keys:_col4, _col3 - Select Operator [SEL_71] (rows=383314495 width=135) - Output:["_col4","_col3"] - Merge Join Operator [MERGEJOIN_187] (rows=383314495 width=135) - Conds:RS_68._col2=RS_69._col0(Inner),Output:["_col3","_col4"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_61] (rows=30 width=2045) - Output:["_col0"] - Filter Operator [FIL_180] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_59] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_68] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_186] (rows=348467716 width=135) - Conds:RS_65._col1=RS_66._col0(Inner),Output:["_col2","_col3","_col4"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Select Operator [SEL_58] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_179] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_56] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=316788826 width=135) - Conds:RS_62._col0=RS_63._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_177] (rows=287989836 width=135) - predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null) - TableScan [TS_50] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_178] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-4-01' AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_53] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col4, _col3 - Merge Join Operator [MERGEJOIN_192] (rows=421645953 width=135) - Conds:RS_94._col4=RS_95._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=105411488 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_46] (rows=105411488 width=135) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Group By Operator [GBY_44] (rows=210822976 width=135) - Output:["_col0"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=210822976 width=135) - Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=28798881 width=106) - Output:["_col0"] - TableScan [TS_12] (rows=28798881 width=106) - default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=191657247 width=135) - Output:["_col0"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=383314495 width=135) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) - Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col3"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=30 width=2045) - Output:["_col0"] - Filter Operator [FIL_175] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_23] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_182] (rows=348467716 width=135) - Conds:RS_29._col1=RS_30._col0(Inner),Output:["_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_174] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_20] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_181] (rows=316788826 width=135) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_172] (rows=287989836 width=135) - predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null) - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_order_number"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_173] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-4-01' AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_191] (rows=383314495 width=135) - Conds:RS_91._col2=RS_92._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=30 width=2045) - Output:["_col0"] - Filter Operator [FIL_170] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_9] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_190] (rows=348467716 width=135) - Conds:RS_88._col1=RS_89._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_169] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_189] (rows=316788826 width=135) - Conds:RS_85._col0=RS_86._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_167] (rows=287989836 width=135) - predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_168] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-4-01' AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 8 diff --git a/ql/src/test/results/clientpositive/perf/query17.q.out b/ql/src/test/results/clientpositive/perf/query17.q.out index f03aeaa..3d3e265 100644 --- a/ql/src/test/results/clientpositive/perf/query17.q.out +++ b/ql/src/test/results/clientpositive/perf/query17.q.out @@ -5,143 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_50] (rows=510205767 width=88) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(VALUE._col0)","avg(VALUE._col1)","stddev_samp(VALUE._col2)","count(VALUE._col3)","avg(VALUE._col4)","stddev_samp(VALUE._col5)","count(VALUE._col6)","avg(VALUE._col7)","stddev_samp(VALUE._col8)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(_col15)","avg(_col15)","stddev_samp(_col15)","count(_col20)","avg(_col20)","stddev_samp(_col20)","count(_col3)","avg(_col3)","stddev_samp(_col3)"],keys:_col24, _col25, _col7 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col24","_col25","_col7","_col15","_col20","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col11=RS_44._col0(Inner),Output:["_col3","_col7","_col15","_col20","_col24","_col25"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col11 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col12, _col11(Inner),Output:["_col3","_col7","_col11","_col15","_col20"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col12, _col11 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col14","_col5","_col9"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col16"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=36524 width=1119) - predicate:((d_quarter_name = '2000Q1') and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query18.q.out b/ql/src/test/results/clientpositive/perf/query18.q.out index cf11954..339f4a7 100644 --- a/ql/src/test/results/clientpositive/perf/query18.q.out +++ b/ql/src/test/results/clientpositive/perf/query18.q.out @@ -5,128 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_49] - Limit [LIM_48] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_47] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_46] - Select Operator [SEL_45] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Group By Operator [GBY_44] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)","avg(VALUE._col4)","avg(VALUE._col5)","avg(VALUE._col6)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_42] (rows=2108229765 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(_col4)","avg(_col5)","avg(_col6)","avg(_col7)","avg(_col8)","avg(_col9)","avg(_col10)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_40] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Merge Join Operator [MERGEJOIN_83] (rows=421645953 width=135) - Conds:RS_37._col0=RS_38._col3(Inner),Output:["_col4","_col6","_col7","_col8","_col11","_col16","_col17","_col18","_col19","_col20","_col26"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col3 - Select Operator [SEL_30] (rows=383314495 width=135) - Output:["_col1","_col10","_col16","_col3","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_82] (rows=383314495 width=135) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col8","_col14","_col16"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_18] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_81] (rows=348467716 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col14"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=465450 width=385) - Output:["_col0","_col3"] - Filter Operator [FIL_76] (rows=465450 width=385) - predicate:((cd_gender = 'M') and (cd_education_status = 'College') and cd_demo_sk is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_education_status","cd_dep_count"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_80] (rows=316788826 width=135) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_75] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_74] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_cdemo_sk is not null and cs_item_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_9] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_bill_cdemo_sk","cs_item_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt","cs_net_profit"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_79] (rows=48400001 width=860) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0","_col4","_col6","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_73] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_78] (rows=44000000 width=860) - Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col0","_col1","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=40000000 width=860) - Output:["_col0","_col1","_col2","_col4"] - Filter Operator [FIL_71] (rows=40000000 width=860) - predicate:((c_birth_month) IN (9, 5, 12, 4, 1, 10) and c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk","c_birth_month","c_birth_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=20000000 width=1014) - predicate:((ca_state) IN ('ND', 'WI', 'AL', 'NC', 'OK', 'MS', 'TN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state","ca_country"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query19.q.out b/ql/src/test/results/clientpositive/perf/query19.q.out index 0ddcd83..9f445cc 100644 --- a/ql/src/test/results/clientpositive/perf/query19.q.out +++ b/ql/src/test/results/clientpositive/perf/query19.q.out @@ -5,117 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_45] - Limit [LIM_44] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_43] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - Select Operator [SEL_40] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_39] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_37] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col7)"],keys:_col10, _col9, _col11, _col12 - Select Operator [SEL_36] (rows=843315281 width=88) - Output:["_col10","_col9","_col11","_col12","_col7"] - Filter Operator [FIL_35] (rows=843315281 width=88) - predicate:(substr(_col17, 1, 5) <> substr(_col19, 1, 5)) - Select Operator [SEL_34] (rows=843315281 width=88) - Output:["_col7","_col9","_col10","_col11","_col12","_col17","_col19"] - Merge Join Operator [MERGEJOIN_73] (rows=843315281 width=88) - Conds:RS_31._col0=RS_32._col2(Inner),Output:["_col3","_col8","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=766650239 width=88) - Output:["_col10","_col11","_col12","_col15","_col2","_col4","_col9"] - Merge Join Operator [MERGEJOIN_72] (rows=766650239 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col2","_col4","_col9","_col10","_col11","_col12","_col15"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_68] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_71] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col3","_col4","_col9","_col10","_col11","_col12"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:((i_manager_id = 7) and i_item_sk is not null) - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id","i_manufact","i_manager_id"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_70] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_66] (rows=18262 width=1119) - predicate:((d_moy = 11) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_65] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_customer_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_69] (rows=88000001 width=860) - Conds:RS_28._col1=RS_29._col0(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query20.q.out b/ql/src/test/results/clientpositive/perf/query20.q.out index c525703..3ba37dd 100644 --- a/ql/src/test/results/clientpositive/perf/query20.q.out +++ b/ql/src/test/results/clientpositive/perf/query20.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_26] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=174233858 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=348467716 width=135) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_38] (rows=348467716 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_36] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_37] (rows=316788826 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_34] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-01-12' AND '2001-02-11' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query21.q.out b/ql/src/test/results/clientpositive/perf/query21.q.out index 0d3ea9d..3afcca1 100644 --- a/ql/src/test/results/clientpositive/perf/query21.q.out +++ b/ql/src/test/results/clientpositive/perf/query21.q.out @@ -55,81 +55,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_29] (rows=12506076 width=15) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col2 > 0)) THEN ((UDFToDouble(_col3) / UDFToDouble(_col2)) BETWEEN 0.6666666666666666 AND 1.5) ELSE (null) END - Group By Operator [GBY_25] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_47] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_44] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_46] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_41] (rows=37584000 width=15) - predicate:(inv_warehouse_sk is not null and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN '1998-03-09' AND '1998-05-07' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query22.q.out b/ql/src/test/results/clientpositive/perf/query22.q.out index 6ff8896..03ceb09 100644 --- a/ql/src/test/results/clientpositive/perf/query22.q.out +++ b/ql/src/test/results/clientpositive/perf/query22.q.out @@ -43,81 +43,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_30] - Limit [LIM_29] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_28] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - Select Operator [SEL_25] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col5"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_22] (rows=250121525 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col3)"],keys:_col8, _col9, _col10, _col11, 0 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col9","_col10","_col11","_col3"] - Merge Join Operator [MERGEJOIN_46] (rows=50024305 width=15) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col8","_col9","_col10","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_43] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=45476640 width=15) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=27 width=1029) - Output:["_col0"] - Filter Operator [FIL_42] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_6] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=37584000 width=15) - predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_41] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query23.q.out b/ql/src/test/results/clientpositive/perf/query23.q.out index 6d4cfca..933fad0 100644 --- a/ql/src/test/results/clientpositive/perf/query23.q.out +++ b/ql/src/test/results/clientpositive/perf/query23.q.out @@ -95,289 +95,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 24 <- Map 23 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 25 <- Map 28 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 31 <- Reducer 30 (SIMPLE_EDGE) -Reducer 4 <- Reducer 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 21 <- Union 5 (CONTAINS), Vertex 20 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_136] - Limit [LIM_135] (rows=1 width=112) - Number of rows:100 - Group By Operator [GBY_133] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Union 5 [SIMPLE_EDGE] - <-Reducer 21 [CONTAINS] - Reduce Output Operator [RS_132] - Group By Operator [GBY_131] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_127] (rows=191667562 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_206] (rows=191667562 width=135) - Conds:RS_124._col2=RS_125._col0(Inner),Output:["_col3","_col4"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_124] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_204] (rows=174243235 width=135) - Conds:RS_121._col1=RS_122._col0(Inner),Output:["_col2","_col3","_col4"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_199] (rows=158402938 width=135) - Conds:RS_118._col0=RS_119._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col0 - Select Operator [SEL_66] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_186] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_64] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_quantity","ws_list_price"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_187] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_67] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Group By Operator [GBY_95] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_91] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_90] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_193] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_89] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Group By Operator [GBY_87] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_85] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_201] (rows=696954748 width=88) - Conds:RS_82._col1=RS_83._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_190] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_200] (rows=633595212 width=88) - Conds:RS_79._col0=RS_80._col0(Inner),Output:["_col1","_col3"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_188] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_70] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Select Operator [SEL_75] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_189] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_73] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Group By Operator [GBY_116] (rows=52799601 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_112] (rows=105599202 width=88) - Output:["_col0"] - Filter Operator [FIL_111] (rows=105599202 width=88) - predicate:(_col1 > 0.95) - Group By Operator [GBY_110] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Group By Operator [GBY_108] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_106] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_202] (rows=633595212 width=88) - Conds:RS_103._col0=RS_104._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_99] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_191] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_97] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Select Operator [SEL_102] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_192] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_100] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_132] - Group By Operator [GBY_131] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_63] (rows=383314495 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_205] (rows=383314495 width=135) - Conds:RS_60._col1=RS_61._col0(Inner),Output:["_col3","_col4"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=52799601 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_48] (rows=105599202 width=88) - Output:["_col0"] - Filter Operator [FIL_47] (rows=105599202 width=88) - predicate:(_col1 > 0.95) - Group By Operator [GBY_46] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Group By Operator [GBY_44] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_42] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_198] (rows=633595212 width=88) - Conds:RS_39._col0=RS_40._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_184] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_33] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_185] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_36] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_203] (rows=348467716 width=135) - Conds:RS_57._col2=RS_58._col0(Inner),Output:["_col1","_col3","_col4"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_27] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_26] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_194] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_25] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_21] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_197] (rows=696954748 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_183] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_196] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_182] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_181] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_195] (rows=316788826 width=135) - Conds:RS_54._col0=RS_55._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_179] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity","cs_list_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_180] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query25.q.out b/ql/src/test/results/clientpositive/perf/query25.q.out index 8e5119c..fa6b185 100644 --- a/ql/src/test/results/clientpositive/perf/query25.q.out +++ b/ql/src/test/results/clientpositive/perf/query25.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col18)","sum(_col23)","sum(_col3)"],keys:_col28, _col29, _col8, _col9 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col28","_col29","_col8","_col9","_col18","_col23","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col14=RS_44._col0(Inner),Output:["_col3","_col8","_col9","_col18","_col23","_col28","_col29"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col14 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col8","_col9","_col14","_col18","_col23"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 4) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_net_loss"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_net_profit"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query26.q.out b/ql/src/test/results/clientpositive/perf/query26.q.out index 6f7aaf7..73205b6 100644 --- a/ql/src/test/results/clientpositive/perf/query26.q.out +++ b/ql/src/test/results/clientpositive/perf/query26.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_33] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col2","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_bill_cdemo_sk is not null and cs_sold_date_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_cdemo_sk","cs_item_sk","cs_promo_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query27.q.out b/ql/src/test/results/clientpositive/perf/query27.q.out index cbd7d29..fe38aa9 100644 --- a/ql/src/test/results/clientpositive/perf/query27.q.out +++ b/ql/src/test/results/clientpositive/perf/query27.q.out @@ -45,96 +45,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_32] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_31] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_29] (rows=2529945843 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(_col2)","avg(_col3)","avg(_col4)","avg(_col5)"],keys:_col0, _col1, 0 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col15","_col17"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_55] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col15"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_store_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_52] (rows=232725 width=385) - predicate:((cd_gender = 'M') and (cd_marital_status = 'U') and (cd_education_status = '2 yr Degree') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query28.q.out b/ql/src/test/results/clientpositive/perf/query28.q.out index cb9d73e..e73f7bc 100644 --- a/ql/src/test/results/clientpositive/perf/query28.q.out +++ b/ql/src/test/results/clientpositive/perf/query28.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT @@ -104,108 +104,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=2497) - Number of rows:100 - Select Operator [SEL_49] (rows=1 width=2497) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Merge Join Operator [MERGEJOIN_58] (rows=1 width=2497) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_46] - Group By Operator [GBY_33] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_30] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_56] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 11 AND 15 and (ss_list_price BETWEEN 66 AND 76 or ss_coupon_amt BETWEEN 920 AND 1920 or ss_wholesale_cost BETWEEN 4 AND 24)) - TableScan [TS_28] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Group By Operator [GBY_40] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_39] - Group By Operator [GBY_38] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_37] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_57] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 6 AND 10 and (ss_list_price BETWEEN 91 AND 101 or ss_coupon_amt BETWEEN 1430 AND 2430 or ss_wholesale_cost BETWEEN 32 AND 52)) - TableScan [TS_35] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - Group By Operator [GBY_5] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_2] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_52] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34)) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_43] - Group By Operator [GBY_12] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_9] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_53] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 26 AND 30 and (ss_list_price BETWEEN 28 AND 38 or ss_coupon_amt BETWEEN 2513 AND 3513 or ss_wholesale_cost BETWEEN 42 AND 62)) - TableScan [TS_7] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_44] - Group By Operator [GBY_19] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_16] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_54] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 21 AND 25 and (ss_list_price BETWEEN 135 AND 145 or ss_coupon_amt BETWEEN 14180 AND 15180 or ss_wholesale_cost BETWEEN 38 AND 58)) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_45] - Group By Operator [GBY_26] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_24] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_23] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_55] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 16 AND 20 and (ss_list_price BETWEEN 142 AND 152 or ss_coupon_amt BETWEEN 3054 AND 4054 or ss_wholesale_cost BETWEEN 80 AND 100)) - TableScan [TS_21] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query29.q.out b/ql/src/test/results/clientpositive/perf/query29.q.out index 587c027..f1fe37a 100644 --- a/ql/src/test/results/clientpositive/perf/query29.q.out +++ b/ql/src/test/results/clientpositive/perf/query29.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col17)","sum(_col22)","sum(_col3)"],keys:_col27, _col28, _col7, _col8 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col27","_col28","_col7","_col8","_col17","_col22","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col13=RS_44._col0(Inner),Output:["_col3","_col7","_col8","_col17","_col22","_col27","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col7","_col8","_col13","_col17","_col22"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 2 AND 5 and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_year) IN (2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query3.q.out b/ql/src/test/results/clientpositive/perf/query3.q.out index 2845a88..acddbe0 100644 --- a/ql/src/test/results/clientpositive/perf/query3.q.out +++ b/ql/src/test/results/clientpositive/perf/query3.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col8, _col4, _col5 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col8","_col4","_col5","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col2","_col4","_col5","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=36524 width=1119) - predicate:((d_moy = 12) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query31.q.out b/ql/src/test/results/clientpositive/perf/query31.q.out index 433d419..4ea7e20 100644 --- a/ql/src/test/results/clientpositive/perf/query31.q.out +++ b/ql/src/test/results/clientpositive/perf/query31.q.out @@ -5,333 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 24 <- Map 28 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE) -Reducer 31 <- Map 34 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Reducer 31 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 38 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 38 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_140] - Select Operator [SEL_139] (rows=316243230 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_138] - Select Operator [SEL_137] (rows=316243230 width=88) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_136] (rows=316243230 width=88) - predicate:CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > (_col5 / _col3))) ELSE ((null > (_col5 / _col3))) END) ELSE (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > null)) ELSE (null) END) END - Merge Join Operator [MERGEJOIN_272] (rows=632486460 width=88) - Conds:RS_132._col6=RS_133._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col7","_col9","_col11"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Group By Operator [GBY_130] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Group By Operator [GBY_128] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_127] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_269] (rows=174243235 width=135) - Conds:RS_124._col1=RS_125._col0(Inner),Output:["_col2","_col7"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Select Operator [SEL_120] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_118] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_124] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_268] (rows=158402938 width=135) - Conds:RS_121._col0=RS_122._col0(Inner),Output:["_col1","_col2"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Select Operator [SEL_114] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_255] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_112] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Select Operator [SEL_117] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_256] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_115] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col6 - Filter Operator [FIL_110] (rows=574987679 width=88) - predicate:CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > (_col3 / _col1))) ELSE ((null > (_col3 / _col1))) END) ELSE (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > null)) ELSE (null) END) END - Select Operator [SEL_109] (rows=1149975359 width=88) - Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_271] (rows=1149975359 width=88) - Conds:RS_104._col0=RS_105._col0(Inner),RS_104._col0=RS_106._col0(Inner),RS_104._col0=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_35] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_261] (rows=696954748 width=88) - Conds:RS_32._col1=RS_33._col0(Inner),Output:["_col2","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_245] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_26] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_260] (rows=633595212 width=88) - Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_243] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_20] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_244] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_58] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Group By Operator [GBY_56] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_55] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_263] (rows=696954748 width=88) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_248] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_46] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_262] (rows=633595212 width=88) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_246] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_40] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_45] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_247] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_43] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_270] (rows=95833780 width=135) - Conds:RS_100._col0=RS_101._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_78] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Group By Operator [GBY_76] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_75] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_265] (rows=174243235 width=135) - Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col7"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_251] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_66] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_264] (rows=158402938 width=135) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_62] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_249] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_60] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_65] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_63] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_98] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Group By Operator [GBY_96] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_95] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_267] (rows=174243235 width=135) - Conds:RS_92._col1=RS_93._col0(Inner),Output:["_col2","_col7"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_254] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_86] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_266] (rows=158402938 width=135) - Conds:RS_89._col0=RS_90._col0(Inner),Output:["_col1","_col2"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_252] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_85] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_253] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_83] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_259] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_242] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_258] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_240] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_241] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query32.q.out b/ql/src/test/results/clientpositive/perf/query32.q.out index c80a4cd..b05b0fa 100644 --- a/ql/src/test/results/clientpositive/perf/query32.q.out +++ b/ql/src/test/results/clientpositive/perf/query32.q.out @@ -39,96 +39,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Group By Operator [GBY_36] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Group By Operator [GBY_34] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col1)"] - Select Operator [SEL_33] (rows=232311810 width=135) - Output:["_col1"] - Filter Operator [FIL_32] (rows=232311810 width=135) - predicate:(_col1 > _col5) - Select Operator [SEL_31] (rows=696935432 width=135) - Output:["_col1","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=696935432 width=135) - Conds:RS_27._col0=RS_28._col0(Inner),RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col5"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=231000 width=1436) - Output:["_col0"] - Filter Operator [FIL_55] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_14] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=158394413 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=158394413 width=135) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=316788826 width=135) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=316788826 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_53] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_59] (rows=316788826 width=135) - Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_57] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_56] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_17] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query33.q.out b/ql/src/test/results/clientpositive/perf/query33.q.out index e42c685..70c94be 100644 --- a/ql/src/test/results/clientpositive/perf/query33.q.out +++ b/ql/src/test/results/clientpositive/perf/query33.q.out @@ -149,289 +149,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:(i_category) IN ('Books') - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:(i_category) IN ('Books') - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:(i_category) IN ('Books') - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query34.q.out b/ql/src/test/results/clientpositive/perf/query34.q.out index 51710bd..8200edd 100644 --- a/ql/src/test/results/clientpositive/perf/query34.q.out +++ b/ql/src/test/results/clientpositive/perf/query34.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 15 AND 20 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County', 'Orange County', 'Appanoose County', 'Franklin Parish', 'Tehama County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.2)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dom BETWEEN 1 AND 3 or d_dom BETWEEN 25 AND 28) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query36.q.out b/ql/src/test/results/clientpositive/perf/query36.q.out index b356628..df4c901 100644 --- a/ql/src/test/results/clientpositive/perf/query36.q.out +++ b/ql/src/test/results/clientpositive/perf/query36.q.out @@ -59,91 +59,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_29] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_28] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_26] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - Group By Operator [GBY_25] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_23] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_51] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col10","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_48] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_50] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_47] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC', 'AL', 'GA') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_49] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_45] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_ext_sales_price","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=36524 width=1119) - predicate:((d_year = 1999) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query39.q.out b/ql/src/test/results/clientpositive/perf/query39.q.out index bde20a6..3b83861 100644 --- a/ql/src/test/results/clientpositive/perf/query39.q.out +++ b/ql/src/test/results/clientpositive/perf/query39.q.out @@ -5,163 +5,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_62] - Select Operator [SEL_61] (rows=13756683 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Select Operator [SEL_59] (rows=13756683 width=15) - Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col8","_col9"] - Merge Join Operator [MERGEJOIN_103] (rows=13756683 width=15) - Conds:RS_56._col1, _col2=RS_57._col1, _col2(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col1, _col2 - Select Operator [SEL_55] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_54] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_53] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_52] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_49] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_102] (rows=50024305 width=15) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_37] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_101] (rows=45476640 width=15) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_95] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_100] (rows=41342400 width=15) - Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_93] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_28] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_33] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_94] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 4) and d_date_sk is not null) - TableScan [TS_31] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col1, _col2 - Select Operator [SEL_27] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_25] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_22] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_99] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_98] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_91] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_97] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_89] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 PREHOOK: query: with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/perf/query40.q.out b/ql/src/test/results/clientpositive/perf/query40.q.out index a98831d..464b630 100644 --- a/ql/src/test/results/clientpositive/perf/query40.q.out +++ b/ql/src/test/results/clientpositive/perf/query40.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_34] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Group By Operator [GBY_29] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col7","_col9","_col11","_col14"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_state"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col7","_col9","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_52] (rows=8116 width=1119) - predicate:(d_date BETWEEN '1998-03-09' AND '1998-05-08' and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col2, _col3=RS_16._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2, _col3 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_order_number","cs_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_51] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_3] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query42.q.out b/ql/src/test/results/clientpositive/perf/query42.q.out index 7100bb3..58e1465 100644 --- a/ql/src/test/results/clientpositive/perf/query42.q.out +++ b/ql/src/test/results/clientpositive/perf/query42.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_23] - Limit [LIM_22] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_33] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_32] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_30] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query43.q.out b/ql/src/test/results/clientpositive/perf/query43.q.out index 72712e8..846ab70 100644 --- a/ql/src/test/results/clientpositive/perf/query43.q.out +++ b/ql/src/test/results/clientpositive/perf/query43.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col5","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=852 width=1910) - predicate:((s_gmt_offset = -6) and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name","s_gmt_offset"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_31] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_day_name"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query45.q.out b/ql/src/test/results/clientpositive/perf/query45.q.out index e2d0da5..0c601f8 100644 --- a/ql/src/test/results/clientpositive/perf/query45.q.out +++ b/ql/src/test/results/clientpositive/perf/query45.q.out @@ -5,117 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 9 <- Reducer 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_47] - Limit [LIM_46] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_45] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_44] - Group By Operator [GBY_42] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Group By Operator [GBY_40] (rows=191667562 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col11)"],keys:_col4, _col3 - Select Operator [SEL_39] (rows=191667562 width=135) - Output:["_col4","_col3","_col11"] - Merge Join Operator [MERGEJOIN_74] (rows=191667562 width=135) - Conds:RS_36._col0=RS_37._col5(Inner),Output:["_col3","_col4","_col11"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_70] (rows=88000001 width=860) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_65] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_73] (rows=174243235 width=135) - Conds:RS_29._col0=RS_30._col1(Inner),Output:["_col5","_col6"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_72] (rows=158402938 width=135) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_68] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_69] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_19] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_71] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_66] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_14] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_11] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:(i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) - TableScan [TS_9] (rows=462000 width=1436) - default@item,i2,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query46.q.out b/ql/src/test/results/clientpositive/perf/query46.q.out index d0275c6..78399d5 100644 --- a/ql/src/test/results/clientpositive/perf/query46.q.out +++ b/ql/src/test/results/clientpositive/perf/query46.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col17, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col17","_col3","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col17"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem', 'Clinton', 'Clifton', 'Springfield') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18263 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=18263 width=1119) - predicate:((d_dow) IN (6, 0) and (d_year) IN (1998, 1999, 2000) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git a/ql/src/test/results/clientpositive/perf/query48.q.out b/ql/src/test/results/clientpositive/perf/query48.q.out index 879cb92..2381cb0 100644 --- a/ql/src/test/results/clientpositive/perf/query48.q.out +++ b/ql/src/test/results/clientpositive/perf/query48.q.out @@ -5,90 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 6 <- Map 9 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_35] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(_col6)"] - Merge Join Operator [MERGEJOIN_58] (rows=15616946 width=88) - Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_50] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=14197224 width=88) - Output:["_col0","_col4"] - Filter Operator [FIL_25] (rows=14197224 width=88) - predicate:(((_col12) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 0 AND 2000) or ((_col12) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 3000) or ((_col12) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 25000)) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col4","_col6","_col12"] - Merge Join Operator [MERGEJOIN_57] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col5","_col7","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=465450 width=385) - Output:["_col0"] - Filter Operator [FIL_54] (rows=465450 width=385) - predicate:((cd_marital_status = 'M') and (cd_education_status = '4 yr Degree') and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col5","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_55] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col3(Inner),Output:["_col1","_col2","_col3","_col5","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_51] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Filter Operator [FIL_52] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 0 AND 2000 or ss_net_profit BETWEEN 150 AND 3000 or ss_net_profit BETWEEN 50 AND 25000) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_net_profit"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query50.q.out b/ql/src/test/results/clientpositive/perf/query50.q.out index 47a00b0..3507155 100644 --- a/ql/src/test/results/clientpositive/perf/query50.q.out +++ b/ql/src/test/results/clientpositive/perf/query50.q.out @@ -117,94 +117,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Group By Operator [GBY_29] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col5","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1704 width=1910) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_55] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_id","s_street_number","s_street_name","s_street_type","s_suite_number","s_city","s_county","s_state","s_zip"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=73049 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col5=RS_19._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col1, _col2, _col4=RS_16._col1, _col2, _col3(Inner),Output:["_col0","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_52] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query51.q.out b/ql/src/test/results/clientpositive/perf/query51.q.out index 55e08d9..9695d99 100644 --- a/ql/src/test/results/clientpositive/perf/query51.q.out +++ b/ql/src/test/results/clientpositive/perf/query51.q.out @@ -87,109 +87,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_53] - Limit [LIM_52] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_51] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_50] - Select Operator [SEL_46] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_58] (rows=116159124 width=88) - predicate:(max_window_0 > max_window_1) - PTF Operator [PTF_45] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END ASC NULLS FIRST","partition by:":"CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END"}] - Select Operator [SEL_44] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END - Merge Join Operator [MERGEJOIN_67] (rows=348477374 width=88) - Conds:RS_40._col0, _col1=RS_41._col0, _col1(Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_36] (rows=79201469 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_32] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_29] (rows=158402938 width=135) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_66] (rows=158402938 width=135) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_61] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_20] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_16] (rows=316797606 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_65] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_59] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_60] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query52.q.out b/ql/src/test/results/clientpositive/perf/query52.q.out index c61ed30..54dd921 100644 --- a/ql/src/test/results/clientpositive/perf/query52.q.out +++ b/ql/src/test/results/clientpositive/perf/query52.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query54.q.out b/ql/src/test/results/clientpositive/perf/query54.q.out index 0b59b1b..5794748 100644 --- a/ql/src/test/results/clientpositive/perf/query54.q.out +++ b/ql/src/test/results/clientpositive/perf/query54.q.out @@ -5,175 +5,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 12 <- Union 13 (CONTAINS) -Map 18 <- Union 13 (CONTAINS) -Reducer 10 <- Reducer 17 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 19 (SIMPLE_EDGE), Union 13 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 13 (CONTAINS) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_69] - Limit [LIM_68] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_67] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - Select Operator [SEL_65] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_64] (rows=174238687 width=88) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_60] (rows=348477374 width=88) - Output:["_col0"] - Group By Operator [GBY_59] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col10 - Select Operator [SEL_56] (rows=696954748 width=88) - Output:["_col10","_col2"] - Merge Join Operator [MERGEJOIN_117] (rows=696954748 width=88) - Conds:RS_53._col1=RS_54._col5(Inner),Output:["_col2","_col10"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_116] (rows=316240138 width=135) - Conds:RS_46._col0=RS_47._col1(Inner),Output:["_col5"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Select Operator [SEL_42] (rows=287491029 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_41] (rows=287491029 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Group By Operator [GBY_39] (rows=574982058 width=135) - Output:["_col0","_col1"],keys:_col10, _col9 - Select Operator [SEL_38] (rows=574982058 width=135) - Output:["_col10","_col9"] - Merge Join Operator [MERGEJOIN_115] (rows=574982058 width=135) - Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col9","_col10"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_26] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=522710951 width=135) - Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col1"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=115500 width=1436) - Output:["_col0"] - Filter Operator [FIL_109] (rows=115500 width=1436) - predicate:((i_category = 'Jewelry') and (i_class = 'football') and i_item_sk is not null) - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_113] (rows=475191764 width=135) - Conds:Union 13._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_108] (rows=18262 width=1119) - predicate:((d_moy = 3) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_106] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_12] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] - <-Map 18 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_15] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_112] (rows=44000000 width=1014) - Conds:RS_43._col1, _col2=RS_44._col0, _col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_105] (rows=1704 width=1910) - predicate:(s_county is not null and s_state is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_county","s_state"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1, _col2 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_104] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null and ca_state is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_111] (rows=633595212 width=88) - Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_102] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_103] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1203 AND 1205 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query55.q.out b/ql/src/test/results/clientpositive/perf/query55.q.out index fb3743d..dc78857 100644 --- a/ql/src/test/results/clientpositive/perf/query55.q.out +++ b/ql/src/test/results/clientpositive/perf/query55.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 36) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query56.q.out b/ql/src/test/results/clientpositive/perf/query56.q.out index 2db6bfb..0035d9f 100644 --- a/ql/src/test/results/clientpositive/perf/query56.q.out +++ b/ql/src/test/results/clientpositive/perf/query56.q.out @@ -133,289 +133,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:(i_color) IN ('orchid', 'chiffon', 'lace') - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:(i_color) IN ('orchid', 'chiffon', 'lace') - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:(i_color) IN ('orchid', 'chiffon', 'lace') - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query58.q.out b/ql/src/test/results/clientpositive/perf/query58.q.out index f7e6a08..e6820ca 100644 --- a/ql/src/test/results/clientpositive/perf/query58.q.out +++ b/ql/src/test/results/clientpositive/perf/query58.q.out @@ -95,270 +95,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 26 <- Map 32 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Map 33 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 28 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_113] - Limit [LIM_112] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_111] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_110] - Select Operator [SEL_109] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_108] (rows=14606 width=135) - predicate:(_col1 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col3 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col5 BETWEEN (0.9 * _col1) AND (1.1 * _col1) and _col5 BETWEEN (0.9 * _col3) AND (1.1 * _col3)) - Merge Join Operator [MERGEJOIN_209] (rows=95833780 width=135) - Conds:RS_104._col0=RS_105._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_102] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_100] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_99] (rows=174243235 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_207] (rows=174243235 width=135) - Conds:RS_96._col4=RS_97._col0(Inner),Output:["_col5","_col7"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_195] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_87] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_206] (rows=158402938 width=135) - Conds:RS_93._col0=RS_94._col0(Inner),Output:["_col4","_col5"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_194] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_84] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_205] (rows=88388 width=1119) - Conds:RS_90._col1=RS_91._col0(Inner),Output:["_col0"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col1 - Select Operator [SEL_73] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_71] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=80353 width=1119) - Conds:RS_80._col1=RS_81._col1(Inner),Output:["_col0"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col1 - Select Operator [SEL_76] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_192] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_74] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col1 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_193] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Filter Operator [FIL_69] (rows=4732408 width=88) - predicate:(_col1 BETWEEN (0.9 * _col3) AND (1.1 * _col3) and _col3 BETWEEN (0.9 * _col1) AND (1.1 * _col1)) - Merge Join Operator [MERGEJOIN_208] (rows=383325119 width=88) - Conds:RS_66._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Group By Operator [GBY_64] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_61] (rows=348467716 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_203] (rows=348467716 width=135) - Conds:RS_58._col4=RS_59._col0(Inner),Output:["_col5","_col7"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_190] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_49] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_202] (rows=316788826 width=135) - Conds:RS_55._col0=RS_56._col0(Inner),Output:["_col4","_col5"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_189] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_46] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_201] (rows=88388 width=1119) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_186] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_33] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_200] (rows=80353 width=1119) - Conds:RS_42._col1=RS_43._col1(Inner),Output:["_col0"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_187] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_188] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_28] (rows=696954748 width=88) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_199] (rows=696954748 width=88) - Conds:RS_25._col4=RS_26._col0(Inner),Output:["_col5","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_185] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_16] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_198] (rows=633595212 width=88) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_184] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_13] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_197] (rows=88388 width=1119) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_181] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_196] (rows=80353 width=1119) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_183] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_182] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 8 diff --git a/ql/src/test/results/clientpositive/perf/query60.q.out b/ql/src/test/results/clientpositive/perf/query60.q.out index a7c8d03..3809247 100644 --- a/ql/src/test/results/clientpositive/perf/query60.q.out +++ b/ql/src/test/results/clientpositive/perf/query60.q.out @@ -155,289 +155,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:(i_category) IN ('Children') - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:(i_category) IN ('Children') - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:(i_category) IN ('Children') - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query64.q.out b/ql/src/test/results/clientpositive/perf/query64.q.out index 468c1bc..115e5e6 100644 --- a/ql/src/test/results/clientpositive/perf/query64.q.out +++ b/ql/src/test/results/clientpositive/perf/query64.q.out @@ -5,657 +5,52 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 12 <- Map 40 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 41 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 34 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 35 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 30 (SIMPLE_EDGE) -Reducer 27 <- Map 31 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 37 <- Map 36 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Reducer 43 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (SIMPLE_EDGE) -Reducer 48 <- Map 47 (SIMPLE_EDGE), Map 53 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 45 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE), Reducer 77 (SIMPLE_EDGE) -Reducer 51 <- Map 79 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 52 <- Map 80 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 55 <- Map 54 (SIMPLE_EDGE), Map 61 (SIMPLE_EDGE) -Reducer 56 <- Map 62 (SIMPLE_EDGE), Reducer 55 (SIMPLE_EDGE) -Reducer 57 <- Map 63 (SIMPLE_EDGE), Reducer 56 (SIMPLE_EDGE) -Reducer 58 <- Reducer 57 (SIMPLE_EDGE), Reducer 68 (SIMPLE_EDGE) -Reducer 59 <- Map 73 (SIMPLE_EDGE), Reducer 58 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 60 <- Map 74 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 65 <- Map 64 (SIMPLE_EDGE), Map 69 (SIMPLE_EDGE) -Reducer 66 <- Map 70 (SIMPLE_EDGE), Reducer 65 (SIMPLE_EDGE) -Reducer 67 <- Map 71 (SIMPLE_EDGE), Reducer 66 (SIMPLE_EDGE) -Reducer 68 <- Map 72 (SIMPLE_EDGE), Reducer 67 (SIMPLE_EDGE) -Reducer 76 <- Map 75 (SIMPLE_EDGE), Map 78 (SIMPLE_EDGE) -Reducer 77 <- Reducer 76 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 52 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE), Vertex 60 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE), Vertex 77 (SORT_PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 79 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 80 (SORT_PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 56 <- Vertex 55 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 57 <- Vertex 56 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 58 <- Vertex 57 (SORT_PARTITION_EDGE), Vertex 68 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 73 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 74 (SORT_PARTITION_EDGE) +Vertex 65 <- Vertex 64 (SORT_PARTITION_EDGE), Vertex 69 (SORT_PARTITION_EDGE) +Vertex 66 <- Vertex 65 (SORT_PARTITION_EDGE), Vertex 70 (SORT_PARTITION_EDGE) +Vertex 67 <- Vertex 66 (SORT_PARTITION_EDGE), Vertex 71 (SORT_PARTITION_EDGE) +Vertex 68 <- Vertex 67 (SORT_PARTITION_EDGE), Vertex 72 (SORT_PARTITION_EDGE) +Vertex 76 <- Vertex 75 (SORT_PARTITION_EDGE), Vertex 78 (SORT_PARTITION_EDGE) +Vertex 77 <- Vertex 76 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_267] - Select Operator [SEL_266] (rows=331415616 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_265] - Select Operator [SEL_264] (rows=331415616 width=88) - Output:["_col0","_col1","_col10","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col2","_col20","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_263] (rows=331415616 width=88) - predicate:(_col30 <= _col13) - Merge Join Operator [MERGEJOIN_658] (rows=994246850 width=88) - Conds:RS_260._col2, _col1, _col3=RS_261._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_260] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_128] (rows=903860754 width=88) - Output:["_col0","_col1","_col10","_col11","_col13","_col14","_col15","_col16","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_127] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_125] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_124] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_656] (rows=1807721509 width=88) - Conds:RS_121._col0=RS_122._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col18 - Select Operator [SEL_117] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_638] (rows=1643383155 width=88) - Conds:RS_114._col21=RS_115._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_602] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_99] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_637] (rows=1493984654 width=88) - Conds:RS_111._col9=RS_112._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_601] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_96] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_636] (rows=1358167838 width=88) - Conds:RS_108._col5=RS_109._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_635] (rows=1234698008 width=88) - Conds:RS_105._col0=RS_106._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col17 - Select Operator [SEL_79] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_633] (rows=1122452711 width=88) - Conds:RS_76._col7=RS_77._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_598] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_70] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_632] (rows=1020411534 width=88) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_597] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_67] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_66] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_65] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_64] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_631] (rows=927646829 width=88) - Conds:RS_61._col0=RS_62._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_626] (rows=106480005 width=860) - Conds:RS_58._col1=RS_59._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_591] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_21] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_625] (rows=96800003 width=860) - Conds:RS_55._col4=RS_56._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_590] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_18] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_624] (rows=88000001 width=860) - Conds:RS_52._col5=RS_53._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col5 - Select Operator [SEL_14] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_588] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_12] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_589] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col4 - Select Operator [SEL_51] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_630] (rows=843315281 width=88) - Conds:RS_48._col3=RS_49._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_596] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_36] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_629] (rows=766650239 width=88) - Conds:RS_45._col6=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_595] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_628] (rows=696954748 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_594] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_30] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_627] (rows=633595212 width=88) - Conds:RS_39._col1, _col8=RS_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1, _col8 - Select Operator [SEL_26] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_592] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_24] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_29] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_593] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_27] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_623] (rows=7920 width=107) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_587] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_9] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_586] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_95] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_94] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_93] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_89] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_634] (rows=316788826 width=135) - Conds:RS_86._col0, _col1=RS_87._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0, _col1 - Select Operator [SEL_82] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_599] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_80] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_600] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_622] (rows=7920 width=107) - Conds:RS_118._col1=RS_119._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_584] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_0] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_585] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_3] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_261] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_258] (rows=903860754 width=88) - Output:["_col1","_col13","_col14","_col15","_col16","_col2","_col3"] - Group By Operator [GBY_257] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_256] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_255] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_254] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_657] (rows=1807721509 width=88) - Conds:RS_251._col0=RS_252._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_251] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_639] (rows=7920 width=107) - Conds:RS_248._col1=RS_249._col0(Inner),Output:["_col0"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_248] - PartitionCols:_col1 - Select Operator [SEL_132] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_603] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_130] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_249] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_604] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_133] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_252] - PartitionCols:_col18 - Select Operator [SEL_247] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_655] (rows=1643383155 width=88) - Conds:RS_244._col21=RS_245._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 80 [SIMPLE_EDGE] - SHUFFLE [RS_245] - PartitionCols:_col0 - Select Operator [SEL_231] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_621] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_229] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_244] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_654] (rows=1493984654 width=88) - Conds:RS_241._col9=RS_242._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 79 [SIMPLE_EDGE] - SHUFFLE [RS_242] - PartitionCols:_col0 - Select Operator [SEL_228] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_620] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_226] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_241] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_653] (rows=1358167838 width=88) - Conds:RS_238._col5=RS_239._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_238] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_652] (rows=1234698008 width=88) - Conds:RS_235._col0=RS_236._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_235] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_640] (rows=7920 width=107) - Conds:RS_232._col1=RS_233._col0(Inner),Output:["_col0"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_232] - PartitionCols:_col1 - Select Operator [SEL_138] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_605] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_136] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 53 [SIMPLE_EDGE] - SHUFFLE [RS_233] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_606] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_139] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_236] - PartitionCols:_col17 - Select Operator [SEL_209] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_650] (rows=1122452711 width=88) - Conds:RS_206._col7=RS_207._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 74 [SIMPLE_EDGE] - SHUFFLE [RS_207] - PartitionCols:_col0 - Select Operator [SEL_202] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_617] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_200] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_206] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_649] (rows=1020411534 width=88) - Conds:RS_203._col1=RS_204._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 73 [SIMPLE_EDGE] - SHUFFLE [RS_204] - PartitionCols:_col0 - Select Operator [SEL_199] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_616] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_197] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 58 [SIMPLE_EDGE] - SHUFFLE [RS_203] - PartitionCols:_col1 - Select Operator [SEL_196] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_195] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_194] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_648] (rows=927646829 width=88) - Conds:RS_191._col0=RS_192._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 57 [SIMPLE_EDGE] - SHUFFLE [RS_191] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_643] (rows=106480005 width=860) - Conds:RS_188._col1=RS_189._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_189] - PartitionCols:_col0 - Select Operator [SEL_153] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_610] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_151] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 56 [SIMPLE_EDGE] - SHUFFLE [RS_188] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_642] (rows=96800003 width=860) - Conds:RS_185._col4=RS_186._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_186] - PartitionCols:_col0 - Select Operator [SEL_150] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_609] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_148] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 55 [SIMPLE_EDGE] - SHUFFLE [RS_185] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_641] (rows=88000001 width=860) - Conds:RS_182._col5=RS_183._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 54 [SIMPLE_EDGE] - SHUFFLE [RS_182] - PartitionCols:_col5 - Select Operator [SEL_144] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_607] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_142] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 61 [SIMPLE_EDGE] - SHUFFLE [RS_183] - PartitionCols:_col0 - Select Operator [SEL_147] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_608] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_145] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 68 [SIMPLE_EDGE] - SHUFFLE [RS_192] - PartitionCols:_col4 - Select Operator [SEL_181] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_647] (rows=843315281 width=88) - Conds:RS_178._col3=RS_179._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 72 [SIMPLE_EDGE] - SHUFFLE [RS_179] - PartitionCols:_col0 - Select Operator [SEL_168] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_615] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_166] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 67 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_646] (rows=766650239 width=88) - Conds:RS_175._col6=RS_176._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 71 [SIMPLE_EDGE] - SHUFFLE [RS_176] - PartitionCols:_col0 - Select Operator [SEL_165] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_614] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_163] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 66 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_645] (rows=696954748 width=88) - Conds:RS_172._col0=RS_173._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 70 [SIMPLE_EDGE] - SHUFFLE [RS_173] - PartitionCols:_col0 - Select Operator [SEL_162] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_613] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_160] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_644] (rows=633595212 width=88) - Conds:RS_169._col1, _col8=RS_170._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col1, _col8 - Select Operator [SEL_156] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_611] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_154] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 69 [SIMPLE_EDGE] - SHUFFLE [RS_170] - PartitionCols:_col0, _col1 - Select Operator [SEL_159] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_612] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_157] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 77 [SIMPLE_EDGE] - SHUFFLE [RS_239] - PartitionCols:_col0 - Select Operator [SEL_225] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_224] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_223] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 76 [SIMPLE_EDGE] - SHUFFLE [RS_222] - PartitionCols:_col0 - Group By Operator [GBY_221] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_219] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_651] (rows=316788826 width=135) - Conds:RS_216._col0, _col1=RS_217._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 75 [SIMPLE_EDGE] - SHUFFLE [RS_216] - PartitionCols:_col0, _col1 - Select Operator [SEL_212] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_618] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_210] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 78 [SIMPLE_EDGE] - SHUFFLE [RS_217] - PartitionCols:_col0, _col1 - Select Operator [SEL_215] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_619] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_213] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query65.q.out b/ql/src/test/results/clientpositive/perf/query65.q.out index 060cddf..af6d33f 100644 --- a/ql/src/test/results/clientpositive/perf/query65.q.out +++ b/ql/src/test/results/clientpositive/perf/query65.q.out @@ -77,125 +77,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_52] - Limit [LIM_51] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_50] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_49] - Select Operator [SEL_48] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_47] (rows=255550079 width=88) - predicate:(_col11 <= (0.1 * _col8)) - Select Operator [SEL_46] (rows=766650239 width=88) - Output:["_col1","_col3","_col4","_col5","_col6","_col8","_col11"] - Merge Join Operator [MERGEJOIN_82] (rows=766650239 width=88) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col4","_col6","_col8","_col9","_col10","_col11"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_78] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_36] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc","i_current_price","i_wholesale_cost","i_brand"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_81] (rows=696954748 width=88) - Conds:RS_39._col0=RS_40._col0(Inner),RS_39._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=158398803 width=88) - Output:["_col0","_col1"] - Group By Operator [GBY_31] (rows=158398803 width=88) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Select Operator [SEL_27] (rows=316797606 width=88) - Output:["_col1","_col2"] - Group By Operator [GBY_26] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_23] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_80] (rows=633595212 width=88) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_79] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_73] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_74] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query66.q.out b/ql/src/test/results/clientpositive/perf/query66.q.out index 5cf60f6..6fd32bb 100644 --- a/ql/src/test/results/clientpositive/perf/query66.q.out +++ b/ql/src/test/results/clientpositive/perf/query66.q.out @@ -439,190 +439,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 22 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 23 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 7 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 9 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Select Operator [SEL_72] (rows=158120068 width=135) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col2","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col3","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col4","_col40","_col41","_col42","_col43","_col5","_col8","_col9"] - Group By Operator [GBY_71] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)","sum(VALUE._col24)","sum(VALUE._col25)","sum(VALUE._col26)","sum(VALUE._col27)","sum(VALUE._col28)","sum(VALUE._col29)","sum(VALUE._col30)","sum(VALUE._col31)","sum(VALUE._col32)","sum(VALUE._col33)","sum(VALUE._col34)","sum(VALUE._col35)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_64] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_62] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_60] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_122] (rows=421645953 width=135) - Conds:RS_57._col3=RS_58._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_114] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_45] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_121] (rows=383314495 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_113] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_42] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_120] (rows=348467716 width=135) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_112] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=316788826 width=135) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_110] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_sold_date_sk is not null and cs_sold_time_sk is not null and cs_ship_mode_sk is not null) - TableScan [TS_33] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_ship_mode_sk","cs_warehouse_sk","cs_quantity","cs_ext_sales_price","cs_net_paid_inc_ship_tax"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_111] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_31] (rows=105417161 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_29] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_27] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_109] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_108] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_9] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_107] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_105] (rows=144002668 width=135) - predicate:(ws_warehouse_sk is not null and ws_sold_date_sk is not null and ws_sold_time_sk is not null and ws_ship_mode_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_ship_mode_sk","ws_warehouse_sk","ws_quantity","ws_sales_price","ws_net_paid_inc_tax"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_106] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] + Vertex 9 diff --git a/ql/src/test/results/clientpositive/perf/query67.q.out b/ql/src/test/results/clientpositive/perf/query67.q.out index 1f38027..2831261 100644 --- a/ql/src/test/results/clientpositive/perf/query67.q.out +++ b/ql/src/test/results/clientpositive/perf/query67.q.out @@ -87,93 +87,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_47] (rows=1149975358 width=88) - predicate:(rank_window_0 <= 100) - PTF Operator [PTF_29] (rows=3449926075 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col16 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_28] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=3449926075 width=88) - Output:["_col0","_col1","_col16","_col2","_col3","_col4","_col5","_col6","_col7"] - Group By Operator [GBY_25] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Group By Operator [GBY_23] (rows=6899852151 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_54] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col7","_col8","_col9","_col11","_col13","_col14","_col15","_col16"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_53] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4","_col7","_col8","_col9","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_48] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_quantity","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col2","_col3","_col4"] - Filter Operator [FIL_49] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq","d_year","d_moy","d_qoy"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query68.q.out b/ql/src/test/results/clientpositive/perf/query68.q.out index b84bdf6..fb1b458 100644 --- a/ql/src/test/results/clientpositive/perf/query68.q.out +++ b/ql/src/test/results/clientpositive/perf/query68.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10","_col11"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col1, _col18, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col18","_col3","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_ext_sales_price","ss_ext_list_price","ss_ext_tax"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git a/ql/src/test/results/clientpositive/perf/query69.q.out b/ql/src/test/results/clientpositive/perf/query69.q.out index 89c3b43..b6962a4 100644 --- a/ql/src/test/results/clientpositive/perf/query69.q.out +++ b/ql/src/test/results/clientpositive/perf/query69.q.out @@ -1,6 +1,6 @@ -Warning: Shuffle Join MERGEJOIN[351][tables = [$hdt$_5, $hdt$_6, $hdt$_4, $hdt$_3]] in Stage 'Reducer 13' is a cross product -Warning: Shuffle Join MERGEJOIN[353][tables = [$hdt$_6, $hdt$_7, $hdt$_5, $hdt$_4]] in Stage 'Reducer 31' is a cross product -Warning: Shuffle Join MERGEJOIN[356][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Reducer 49' is a cross product +Warning: Shuffle Join MERGEJOIN[351][tables = [$hdt$_5, $hdt$_6, $hdt$_4, $hdt$_3]] in Stage 'Vertex 13' is a cross product +Warning: Shuffle Join MERGEJOIN[353][tables = [$hdt$_6, $hdt$_7, $hdt$_5, $hdt$_4]] in Stage 'Vertex 31' is a cross product +Warning: Shuffle Join MERGEJOIN[356][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Vertex 49' is a cross product PREHOOK: query: explain select cd_gender, cd_marital_status, @@ -94,498 +94,46 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 26 (SIMPLE_EDGE) -Reducer 24 <- Map 27 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (SIMPLE_EDGE) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE) -Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 32 <- Reducer 31 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 38 (SIMPLE_EDGE) -Reducer 36 <- Map 39 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) -Reducer 37 <- Reducer 36 (SIMPLE_EDGE) -Reducer 4 <- Reducer 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 41 <- Map 40 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE) -Reducer 42 <- Map 45 (SIMPLE_EDGE), Reducer 41 (SIMPLE_EDGE) -Reducer 43 <- Reducer 42 (SIMPLE_EDGE) -Reducer 47 <- Map 46 (SIMPLE_EDGE), Map 51 (SIMPLE_EDGE) -Reducer 48 <- Reducer 47 (SIMPLE_EDGE), Reducer 55 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE), Reducer 61 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE) -Reducer 53 <- Map 52 (SIMPLE_EDGE), Map 56 (SIMPLE_EDGE) -Reducer 54 <- Map 57 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE) -Reducer 55 <- Reducer 54 (SIMPLE_EDGE) -Reducer 59 <- Map 58 (SIMPLE_EDGE), Map 62 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 60 <- Map 63 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 61 <- Reducer 60 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 43 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 41 <- Vertex 40 (SORT_PARTITION_EDGE), Vertex 44 (SORT_PARTITION_EDGE) +Vertex 42 <- Vertex 41 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE) +Vertex 47 <- Vertex 46 (SORT_PARTITION_EDGE), Vertex 51 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 50 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (SORT_PARTITION_EDGE), Vertex 57 (SORT_PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 61 <- Vertex 60 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_214] - Limit [LIM_213] (rows=100 width=248) - Number of rows:100 - Select Operator [SEL_212] (rows=5102057559316637 width=248) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_211] - Select Operator [SEL_210] (rows=5102057559316637 width=248) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Group By Operator [GBY_209] (rows=5102057559316637 width=248) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_208] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_207] (rows=10204115118633274 width=248) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count()"],keys:_col6, _col7, _col8, _col9, _col10 - Select Operator [SEL_206] (rows=10204115118633274 width=248) - Output:["_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_205] (rows=10204115118633274 width=248) - predicate:_col15 is null - Merge Join Operator [MERGEJOIN_357] (rows=20408230237266548 width=248) - Conds:RS_202._col0=RS_203._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col15"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_202] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=18552936177209164 width=248) - Output:["_col0","_col10","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_140] (rows=18552936177209164 width=248) - predicate:_col13 is null - Select Operator [SEL_139] (rows=37105872354418328 width=248) - Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col13"] - Merge Join Operator [MERGEJOIN_354] (rows=37105872354418328 width=248) - Conds:RS_135._col0=RS_136._col0(Left Outer),RS_135._col0=RS_137._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_136] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=4216686374121617 width=996) - Output:["_col0","_col1"] - Group By Operator [GBY_67] (rows=4216686374121617 width=996) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=8433372748243235 width=996) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_351] (rows=8433372748243235 width=996) - Conds:(Inner),Output:["_col6"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_61] - Merge Join Operator [MERGEJOIN_350] (rows=174243235 width=135) - Conds:RS_58._col1=RS_59._col0(Inner) - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_335] (rows=158402938 width=135) - Conds:RS_55._col0=RS_56._col0(Inner),Output:["_col1"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_309] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_310] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_337] (rows=96800003 width=860) - Conds:RS_27._col1=RS_28._col0(Inner),Output:["_col0"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_313] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_21] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_336] (rows=88000001 width=860) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col0","_col1"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Select Operator [SEL_17] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_311] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_15] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_312] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_62] - Group By Operator [GBY_53] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Group By Operator [GBY_51] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_339] (rows=96800003 width=860) - Conds:RS_47._col1=RS_48._col0(Inner),Output:["_col0"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_43] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_316] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_41] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_338] (rows=88000001 width=860) - Conds:RS_44._col2=RS_45._col0(Inner),Output:["_col0","_col1"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col2 - Select Operator [SEL_37] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_314] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_35] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_315] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_38] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_334] (rows=96800003 width=860) - Conds:RS_132._col1=RS_133._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_308] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status","cd_purchase_estimate","cd_credit_rating"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_333] (rows=88000001 width=860) - Conds:RS_129._col2=RS_130._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_306] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_130] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_307] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_137] - PartitionCols:_col0 - Group By Operator [GBY_127] (rows=16866305250077374 width=273) - Output:["_col0"],keys:KEY._col0 - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Group By Operator [GBY_125] (rows=33732610500154748 width=273) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_353] (rows=33732610500154748 width=273) - Conds:(Inner),Output:["_col6"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_121] - Merge Join Operator [MERGEJOIN_352] (rows=696954748 width=88) - Conds:RS_118._col1=RS_119._col0(Inner) - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_340] (rows=633595212 width=88) - Conds:RS_115._col0=RS_116._col0(Inner),Output:["_col1"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_71] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_317] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_69] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_74] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_318] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_72] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Group By Operator [GBY_93] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_342] (rows=96800003 width=860) - Conds:RS_87._col1=RS_88._col0(Inner),Output:["_col0"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Select Operator [SEL_83] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_321] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_81] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_341] (rows=88000001 width=860) - Conds:RS_84._col2=RS_85._col0(Inner),Output:["_col0","_col1"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col2 - Select Operator [SEL_77] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_319] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_75] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 38 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col0 - Select Operator [SEL_80] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_320] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_78] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_122] - Group By Operator [GBY_113] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Group By Operator [GBY_111] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_344] (rows=96800003 width=860) - Conds:RS_107._col1=RS_108._col0(Inner),Output:["_col0"] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_103] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_324] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_101] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 41 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_343] (rows=88000001 width=860) - Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col0","_col1"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col2 - Select Operator [SEL_97] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_322] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_95] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 44 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_323] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_98] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_203] - PartitionCols:_col0 - Select Operator [SEL_201] (rows=8432918901433858 width=546) - Output:["_col0","_col1"] - Group By Operator [GBY_200] (rows=8432918901433858 width=546) - Output:["_col0"],keys:KEY._col0 - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_199] - PartitionCols:_col0 - Group By Operator [GBY_198] (rows=16865837802867716 width=546) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_356] (rows=16865837802867716 width=546) - Conds:(Inner),Output:["_col6"] - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_194] - Merge Join Operator [MERGEJOIN_355] (rows=348467716 width=135) - Conds:RS_191._col1=RS_192._col0(Inner) - <-Reducer 47 [SIMPLE_EDGE] - SHUFFLE [RS_191] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_345] (rows=316788826 width=135) - Conds:RS_188._col0=RS_189._col0(Inner),Output:["_col1"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_188] - PartitionCols:_col0 - Select Operator [SEL_144] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_325] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_142] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_customer_sk"] - <-Map 51 [SIMPLE_EDGE] - SHUFFLE [RS_189] - PartitionCols:_col0 - Select Operator [SEL_147] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_326] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_145] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 55 [SIMPLE_EDGE] - SHUFFLE [RS_192] - PartitionCols:_col0 - Group By Operator [GBY_166] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 54 [SIMPLE_EDGE] - SHUFFLE [RS_165] - PartitionCols:_col0 - Group By Operator [GBY_164] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_347] (rows=96800003 width=860) - Conds:RS_160._col1=RS_161._col0(Inner),Output:["_col0"] - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_161] - PartitionCols:_col0 - Select Operator [SEL_156] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_329] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_154] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_160] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_346] (rows=88000001 width=860) - Conds:RS_157._col2=RS_158._col0(Inner),Output:["_col0","_col1"] - <-Map 52 [SIMPLE_EDGE] - SHUFFLE [RS_157] - PartitionCols:_col2 - Select Operator [SEL_150] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_327] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_148] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_158] - PartitionCols:_col0 - Select Operator [SEL_153] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_328] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_151] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 61 [SIMPLE_EDGE] - SHUFFLE [RS_195] - Group By Operator [GBY_186] (rows=48400001 width=860) - Output:["_col0"],keys:KEY._col0 - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_185] - PartitionCols:_col0 - Group By Operator [GBY_184] (rows=96800003 width=860) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_349] (rows=96800003 width=860) - Conds:RS_180._col1=RS_181._col0(Inner),Output:["_col0"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_181] - PartitionCols:_col0 - Select Operator [SEL_176] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_332] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_174] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_180] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_348] (rows=88000001 width=860) - Conds:RS_177._col2=RS_178._col0(Inner),Output:["_col0","_col1"] - <-Map 58 [SIMPLE_EDGE] - SHUFFLE [RS_177] - PartitionCols:_col2 - Select Operator [SEL_170] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_330] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_168] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col0 - Select Operator [SEL_173] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_331] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_171] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query7.q.out b/ql/src/test/results/clientpositive/perf/query7.q.out index 65478ab..18ae02d 100644 --- a/ql/src/test/results/clientpositive/perf/query7.q.out +++ b/ql/src/test/results/clientpositive/perf/query7.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_promo_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query70.q.out b/ql/src/test/results/clientpositive/perf/query70.q.out index b4eaee6..b1e4711 100644 --- a/ql/src/test/results/clientpositive/perf/query70.q.out +++ b/ql/src/test/results/clientpositive/perf/query70.q.out @@ -75,151 +75,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 12 <- Map 17 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_64] - Limit [LIM_63] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_62] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_61] - Select Operator [SEL_59] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_58] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_57] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_55] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_54] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_52] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_50] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_92] (rows=766650239 width=88) - Conds:RS_47._col7=RS_48._col0(Inner),Output:["_col2","_col6","_col7"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Group By Operator [GBY_39] (rows=58079562 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=116159124 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_32] (rows=116159124 width=88) - Output:["_col0"] - Filter Operator [FIL_84] (rows=116159124 width=88) - predicate:(rank_window_0 <= 5) - PTF Operator [PTF_31] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_30] (rows=348477374 width=88) - Output:["_col0","_col1"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col6 - Select Operator [SEL_24] (rows=696954748 width=88) - Output:["_col6","_col2"] - Merge Join Operator [MERGEJOIN_91] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col6"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=575995635 width=88) - predicate:(ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_9] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_86] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_89] (rows=696954748 width=88) - Conds:RS_44._col1=RS_45._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_6] (rows=1704 width=1910) - default@store,s,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=633595212 width=88) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_82] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query71.q.out b/ql/src/test/results/clientpositive/perf/query71.q.out index 9bb010c..cce17b4 100644 --- a/ql/src/test/results/clientpositive/perf/query71.q.out +++ b/ql/src/test/results/clientpositive/perf/query71.q.out @@ -5,134 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 15 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Map 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 3 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_53] - Select Operator [SEL_52] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_49] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_48] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_46] (rows=1341632299 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col0)"],keys:_col4, _col8, _col9, _col5 - Select Operator [SEL_45] (rows=1341632299 width=108) - Output:["_col4","_col8","_col9","_col5","_col0"] - Merge Join Operator [MERGEJOIN_87] (rows=1341632299 width=108) - Conds:RS_42._col2=RS_43._col0(Inner),Output:["_col0","_col4","_col5","_col8","_col9"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=86400 width=471) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=86400 width=471) - predicate:(((t_meal_time = 'breakfast') or (t_meal_time = 'dinner')) and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute","t_meal_time"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_86] (rows=1219665700 width=108) - Conds:Union 3._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_33] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_84] (rows=316788826 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_78] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_77] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null and cs_sold_time_sk is not null) - TableScan [TS_10] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_31] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_79] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_sold_time_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_80] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_83] (rows=158402938 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null and ws_sold_time_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query72.q.out b/ql/src/test/results/clientpositive/perf/query72.q.out index dff4c3a..e4a2f93 100644 --- a/ql/src/test/results/clientpositive/perf/query72.q.out +++ b/ql/src/test/results/clientpositive/perf/query72.q.out @@ -5,192 +5,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 22 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 23 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 20 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 21 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 13 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Group By Operator [GBY_71] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_69] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col3)","count(_col4)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_67] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_143] (rows=82996844 width=135) - Conds:RS_64._col4, _col6=RS_65._col0, _col1(Left Outer),Output:["_col13","_col15","_col22","_col28"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_60] (rows=28798881 width=106) - Output:["_col0","_col1"] - Filter Operator [FIL_133] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_58] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col4, _col6 - Merge Join Operator [MERGEJOIN_142] (rows=75451675 width=135) - Conds:RS_61._col5=RS_62._col0(Left Outer),Output:["_col4","_col6","_col13","_col15","_col22","_col28"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=2300 width=1179) - Output:["_col0"] - TableScan [TS_56] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col5 - Select Operator [SEL_55] (rows=68592431 width=135) - Output:["_col13","_col15","_col22","_col4","_col5","_col6"] - Filter Operator [FIL_54] (rows=68592431 width=135) - predicate:(UDFToDouble(_col27) > (UDFToDouble(_col21) + 5.0)) - Select Operator [SEL_53] (rows=205777294 width=135) - Output:["_col4","_col5","_col6","_col13","_col15","_col21","_col22","_col27"] - Merge Join Operator [MERGEJOIN_141] (rows=205777294 width=135) - Conds:RS_50._col10=RS_51._col0(Inner),Output:["_col4","_col5","_col6","_col13","_col14","_col23","_col25","_col27"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_131] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_29] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col10 - Merge Join Operator [MERGEJOIN_140] (rows=187070264 width=135) - Conds:RS_47._col1=RS_48._col0(Inner),Output:["_col4","_col5","_col6","_col10","_col13","_col14","_col23","_col25"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_130] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_26] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_139] (rows=170063873 width=135) - Conds:RS_44._col4=RS_45._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14","_col23"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_129] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_138] (rows=154603518 width=135) - Conds:RS_41._col3=RS_42._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_128] (rows=3600 width=107) - predicate:((hd_buy_potential = '1001-5000') and hd_demo_sk is not null) - TableScan [TS_20] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_137] (rows=140548650 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=930900 width=385) - Output:["_col0"] - Filter Operator [FIL_127] (rows=930900 width=385) - predicate:((cd_marital_status = 'M') and cd_demo_sk is not null) - TableScan [TS_17] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_136] (rows=127771498 width=135) - Conds:RS_35._col8, _col14=RS_36._col0, _col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Select Operator [SEL_16] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_week_seq is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col8, _col14 - Merge Join Operator [MERGEJOIN_135] (rows=116155905 width=135) - Conds:RS_32._col0=RS_33._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col10","_col13","_col14"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=36524 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) - TableScan [TS_11] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_week_seq","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=105596275 width=135) - Output:["_col0","_col1","_col10","_col2","_col3","_col4","_col5","_col6","_col8"] - Filter Operator [FIL_9] (rows=105596275 width=135) - predicate:(_col11 < _col7) - Merge Join Operator [MERGEJOIN_134] (rows=316788826 width=135) - Conds:RS_6._col4=RS_7._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col4 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_sold_date_sk is not null and cs_ship_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_date_sk","cs_bill_cdemo_sk","cs_bill_hdemo_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_quantity"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_124] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_3] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] + Vertex 13 diff --git a/ql/src/test/results/clientpositive/perf/query73.q.out b/ql/src/test/results/clientpositive/perf/query73.q.out index 9e1a39a..5daf3d9 100644 --- a/ql/src/test/results/clientpositive/perf/query73.q.out +++ b/ql/src/test/results/clientpositive/perf/query73.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 1 AND 5 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.0)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query75.q.out b/ql/src/test/results/clientpositive/perf/query75.q.out index f78ce8a..b93f79e 100644 --- a/ql/src/test/results/clientpositive/perf/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/query75.q.out @@ -5,387 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 21 <- Map 24 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Map 25 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 34 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Union 30 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Map 41 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Map 47 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 48 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 31 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 5 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 5 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 30 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Union 30 (CONTAINS), Vertex 37 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 45 <- Union 30 (CONTAINS), Vertex 44 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_156] - Limit [LIM_155] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_154] (rows=245965926 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_153] - Select Operator [SEL_152] (rows=245965926 width=108) - Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_151] (rows=245965926 width=108) - predicate:((CAST( _col4 AS decimal(17,2)) / CAST( _col10 AS decimal(17,2))) < 0.9) - Merge Join Operator [MERGEJOIN_259] (rows=737897778 width=108) - Conds:RS_148._col0, _col1, _col2, _col3=RS_149._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_146] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 30 [SIMPLE_EDGE] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_95] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_252] (rows=383314495 width=135) - Conds:RS_92._col1, _col2=RS_93._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_232] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_251] (rows=348467716 width=135) - Conds:RS_89._col1=RS_90._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_231] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_80] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_250] (rows=316788826 width=135) - Conds:RS_86._col0=RS_87._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Select Operator [SEL_76] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_229] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_74] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_230] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_117] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_255] (rows=766650239 width=88) - Conds:RS_114._col1, _col2=RS_115._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_236] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_105] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_254] (rows=696954748 width=88) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_104] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_235] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_102] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_253] (rows=633595212 width=88) - Conds:RS_108._col0=RS_109._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_233] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_96] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_234] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_99] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 45 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_141] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_258] (rows=191667562 width=135) - Conds:RS_138._col1, _col2=RS_139._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_139] - PartitionCols:_col0, _col1 - Select Operator [SEL_131] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_240] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_129] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_138] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_257] (rows=174243235 width=135) - Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_136] - PartitionCols:_col0 - Select Operator [SEL_128] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_239] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_126] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_256] (rows=158402938 width=135) - Conds:RS_132._col0=RS_133._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col0 - Select Operator [SEL_122] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_237] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_120] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Select Operator [SEL_125] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_238] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_123] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_72] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_43] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_246] (rows=766650239 width=88) - Conds:RS_40._col1, _col2=RS_41._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_33] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_224] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_31] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_245] (rows=696954748 width=88) - Conds:RS_37._col1=RS_38._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_223] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_28] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_244] (rows=633595212 width=88) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_221] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_222] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_67] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_249] (rows=191667562 width=135) - Conds:RS_64._col1, _col2=RS_65._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_57] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_228] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_55] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_248] (rows=174243235 width=135) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_227] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_52] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_247] (rows=158402938 width=135) - Conds:RS_58._col0=RS_59._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_225] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_46] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_226] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_49] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_21] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_243] (rows=383314495 width=135) - Conds:RS_18._col1, _col2=RS_19._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_220] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_9] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_242] (rows=348467716 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_219] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_241] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_217] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_218] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 8 diff --git a/ql/src/test/results/clientpositive/perf/query76.q.out b/ql/src/test/results/clientpositive/perf/query76.q.out index c7dbb37..8f1d85c 100644 --- a/ql/src/test/results/clientpositive/perf/query76.q.out +++ b/ql/src/test/results/clientpositive/perf/query76.q.out @@ -5,151 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 16 <- Map 18 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_57] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Group By Operator [GBY_54] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_31] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_87] (rows=87121617 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_80] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_22] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=79201469 width=135) - Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_19] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col1 - Select Operator [SEL_18] (rows=72001334 width=135) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_78] (rows=72001334 width=135) - predicate:(ws_web_page_sk is null and ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_ext_sales_price"] - <-Reducer 16 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_49] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_89] (rows=174233858 width=135) - Conds:RS_46._col0=RS_47._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_40] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_88] (rows=158394413 width=135) - Conds:RS_43._col2=RS_44._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col2 - Select Operator [SEL_36] (rows=143994918 width=135) - Output:["_col0","_col2","_col3"] - Filter Operator [FIL_81] (rows=143994918 width=135) - predicate:(cs_warehouse_sk is null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_34] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_82] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Reducer 3 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_15] (rows=348477373 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_85] (rows=348477373 width=88) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col1","_col5","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_77] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_84] (rows=316797605 width=88) - Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=287997817 width=88) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_76] (rows=287997817 width=88) - predicate:(ss_addr_sk is null and ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query79.q.out b/ql/src/test/results/clientpositive/perf/query79.q.out index 6ce9592..fc79760 100644 --- a/ql/src/test/results/clientpositive/perf/query79.q.out +++ b/ql/src/test/results/clientpositive/perf/query79.q.out @@ -5,98 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=421657640 width=88) - Conds:RS_30._col0=RS_31._col1(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col3, _col5, _col13 - Select Operator [SEL_24] (rows=766650239 width=88) - Output:["_col1","_col3","_col5","_col13","_col6","_col7"] - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col13"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_56] (rows=6000 width=107) - predicate:(((hd_dep_count = 8) or (hd_vehicle_count > 0)) and hd_demo_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col4=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=189 width=1910) - Output:["_col0","_col2"] - Filter Operator [FIL_55] (rows=189 width=1910) - predicate:(s_number_employees BETWEEN 200 AND 295 and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_number_employees","s_city"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=18262 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dow = 1) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query80.q.out b/ql/src/test/results/clientpositive/perf/query80.q.out index 10ec36a..b4a88a7 100644 --- a/ql/src/test/results/clientpositive/perf/query80.q.out +++ b/ql/src/test/results/clientpositive/perf/query80.q.out @@ -5,318 +5,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 25 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 27 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 36 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 31 <- Map 37 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 38 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 33 <- Map 39 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 11 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 8 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 34 <- Union 8 (CONTAINS), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 8 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 10 - File Output Operator [FS_128] - Limit [LIM_127] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_126] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_125] - Select Operator [SEL_124] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_123] (rows=1217531358 width=108) - Output:["_col0","_col1","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_77] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_76] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Group By Operator [GBY_74] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_72] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_213] (rows=463810558 width=135) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=46000 width=460) - Output:["_col0","_col1"] - Filter Operator [FIL_197] (rows=46000 width=460) - predicate:cp_catalog_page_sk is not null - TableScan [TS_54] (rows=46000 width=460) - default@catalog_page,catalog_page,Tbl:COMPLETE,Col:NONE,Output:["cp_catalog_page_sk","cp_catalog_page_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_212] (rows=421645953 width=135) - Conds:RS_66._col3=RS_67._col0(Inner),Output:["_col1","_col5","_col6","_col9","_col10"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_196] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_51] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_211] (rows=383314495 width=135) - Conds:RS_63._col2=RS_64._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col9","_col10"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_195] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_48] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_210] (rows=348467716 width=135) - Conds:RS_60._col0=RS_61._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_194] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-04 and d_date_sk is not null) - TableScan [TS_45] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_209] (rows=316788826 width=135) - Conds:RS_57._col2, _col4=RS_58._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col2, _col4 - Select Operator [SEL_41] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_192] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_catalog_page_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_39] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_catalog_page_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_ext_sales_price","cs_net_profit"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0, _col1 - Select Operator [SEL_44] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_193] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_42] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_amount","cr_net_loss"] - <-Reducer 34 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_118] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_117] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_113] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_218] (rows=231917759 width=135) - Conds:RS_110._col2=RS_111._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col0 - Select Operator [SEL_97] (rows=84 width=1850) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=84 width=1850) - predicate:web_site_sk is not null - TableScan [TS_95] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_site_id"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_217] (rows=210834322 width=135) - Conds:RS_107._col3=RS_108._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 38 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_202] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_92] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_216] (rows=191667562 width=135) - Conds:RS_104._col1=RS_105._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_201] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_215] (rows=174243235 width=135) - Conds:RS_101._col0=RS_102._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_200] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-04 and d_date_sk is not null) - TableScan [TS_86] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_214] (rows=158402938 width=135) - Conds:RS_98._col1, _col4=RS_99._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1, _col4 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_198] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_web_site_sk is not null and ws_item_sk is not null and ws_promo_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_site_sk","ws_promo_sk","ws_order_number","ws_ext_sales_price","ws_net_profit"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_199] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_83] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_38] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_37] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Group By Operator [GBY_35] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_33] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_208] (rows=927646829 width=88) - Conds:RS_30._col2=RS_31._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_207] (rows=843315281 width=88) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_190] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_12] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_206] (rows=766650239 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_189] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_205] (rows=696954748 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_188] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-04 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=633595212 width=88) - Conds:RS_18._col1, _col4=RS_19._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_186] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_ext_sales_price","ss_net_profit"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_187] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_amt","sr_net_loss"] + Vertex 10 diff --git a/ql/src/test/results/clientpositive/perf/query82.q.out b/ql/src/test/results/clientpositive/perf/query82.q.out index 6f784c4..70130d2 100644 --- a/ql/src/test/results/clientpositive/perf/query82.q.out +++ b/ql/src/test/results/clientpositive/perf/query82.q.out @@ -5,71 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_26] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_23] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_21] (rows=1267190424 width=88) - Output:["_col0","_col1","_col2"],keys:_col2, _col3, _col4 - Merge Join Operator [MERGEJOIN_44] (rows=1267190424 width=88) - Conds:RS_16._col0=RS_17._col0(Inner),RS_17._col0=RS_18._col1(Inner),Output:["_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:ss_item_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25666 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=25666 width=1436) - predicate:((i_manufact_id) IN (437, 129, 727, 663) and i_current_price BETWEEN 30 AND 60 and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=4593600 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4176000 width=15) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=4176000 width=15) - predicate:(inv_quantity_on_hand BETWEEN 100 AND 500 and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_6] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2002-05-30' AND '2002-07-30' and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 4 diff --git a/ql/src/test/results/clientpositive/perf/query83.q.out b/ql/src/test/results/clientpositive/perf/query83.q.out index 2789643..43c5f24 100644 --- a/ql/src/test/results/clientpositive/perf/query83.q.out +++ b/ql/src/test/results/clientpositive/perf/query83.q.out @@ -133,308 +133,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE) -Reducer 28 <- Map 27 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 36 <- Reducer 35 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 18 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_137] - Limit [LIM_136] (rows=100 width=77) - Number of rows:100 - Select Operator [SEL_135] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_134] - Select Operator [SEL_133] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_231] (rows=76653825 width=77) - Conds:RS_129._col0=RS_130._col0(Inner),RS_129._col0=RS_131._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_130] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=34842647 width=77) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=69685294 width=77) - Output:["_col0","_col1"],aggregations:["sum(_col0)"],keys:_col1 - Filter Operator [FIL_80] (rows=69685294 width=77) - predicate:_col1 is not null - Select Operator [SEL_79] (rows=69685294 width=77) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_229] (rows=69685294 width=77) - Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col2","_col4"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_224] (rows=63350266 width=77) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=57591150 width=77) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_209] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_returned_date_sk is not null) - TableScan [TS_43] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_return_quantity"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_210] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_46] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_225] (rows=80353 width=1119) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col0"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_211] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_49] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Group By Operator [GBY_67] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_220] (rows=80353 width=1119) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col0"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_53] (rows=73049 width=1119) - Output:["_col0","_col1"] - TableScan [TS_52] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Group By Operator [GBY_59] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_56] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_213] (rows=36525 width=1119) - predicate:(d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') - TableScan [TS_54] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_131] - PartitionCols:_col0 - Group By Operator [GBY_127] (rows=8711072 width=92) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Group By Operator [GBY_125] (rows=17422145 width=92) - Output:["_col0","_col1"],aggregations:["sum(_col0)"],keys:_col1 - Filter Operator [FIL_123] (rows=17422145 width=92) - predicate:_col1 is not null - Select Operator [SEL_122] (rows=17422145 width=92) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_230] (rows=17422145 width=92) - Conds:RS_119._col0=RS_120._col0(Inner),Output:["_col2","_col4"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_226] (rows=15838314 width=92) - Conds:RS_116._col1=RS_117._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col1 - Select Operator [SEL_88] (rows=14398467 width=92) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_214] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_returned_date_sk is not null) - TableScan [TS_86] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_return_quantity"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_215] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_120] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_227] (rows=80353 width=1119) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Select Operator [SEL_94] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_216] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_92] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Group By Operator [GBY_110] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Group By Operator [GBY_108] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_221] (rows=80353 width=1119) - Conds:RS_104._col1=RS_105._col0(Inner),Output:["_col0"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col1 - Select Operator [SEL_96] (rows=73049 width=1119) - Output:["_col0","_col1"] - TableScan [TS_95] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_102] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_100] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_99] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_218] (rows=36525 width=1119) - predicate:(d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') - TableScan [TS_97] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=17423323 width=106) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Group By Operator [GBY_39] (rows=34846646 width=106) - Output:["_col0","_col1"],aggregations:["sum(_col0)"],keys:_col1 - Filter Operator [FIL_37] (rows=34846646 width=106) - predicate:_col1 is not null - Select Operator [SEL_36] (rows=34846646 width=106) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_228] (rows=34846646 width=106) - Conds:RS_33._col0=RS_34._col0(Inner),Output:["_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_222] (rows=31678769 width=106) - Conds:RS_30._col1=RS_31._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_204] (rows=28798881 width=106) - predicate:(cr_item_sk is not null and cr_returned_date_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_item_sk","cr_return_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_205] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_223] (rows=80353 width=1119) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_206] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Group By Operator [GBY_22] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_219] (rows=80353 width=1119) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col0"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_10] (rows=73049 width=1119) - Output:["_col0","_col1"] - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Group By Operator [GBY_14] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_13] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_208] (rows=36525 width=1119) - predicate:(d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') - TableScan [TS_11] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query84.q.out b/ql/src/test/results/clientpositive/perf/query84.q.out index cf68729..8ccd524 100644 --- a/ql/src/test/results/clientpositive/perf/query84.q.out +++ b/ql/src/test/results/clientpositive/perf/query84.q.out @@ -5,95 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 7 <- Reducer 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=860) - Number of rows:100 - Select Operator [SEL_35] (rows=212960011 width=860) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=212960011 width=860) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_67] (rows=212960011 width=860) - Conds:RS_29._col0=RS_30._col0(Inner),RS_30._col0=RS_31._col1(Inner),Output:["_col2","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0"] - Filter Operator [FIL_58] (rows=57591150 width=77) - predicate:sr_cdemo_sk is not null - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_cdemo_sk"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_59] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_66] (rows=96800003 width=860) - Conds:RS_25._col2=RS_26._col0(Inner),Output:["_col0","_col1","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_65] (rows=7920 width=107) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=2 width=12) - Output:["_col0"] - Filter Operator [FIL_63] (rows=2 width=12) - predicate:((ib_lower_bound >= 32287) and (ib_upper_bound <= 82287) and ib_income_band_sk is not null) - TableScan [TS_15] (rows=20 width=12) - default@income_band,income_band,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk","ib_lower_bound","ib_upper_bound"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_64] (rows=88000001 width=860) - Conds:RS_22._col3=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_60] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_id","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_61] (rows=20000000 width=1014) - predicate:((ca_city = 'Hopewell') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 3 diff --git a/ql/src/test/results/clientpositive/perf/query85.q.out b/ql/src/test/results/clientpositive/perf/query85.q.out index e7f7348..45918ea 100644 --- a/ql/src/test/results/clientpositive/perf/query85.q.out +++ b/ql/src/test/results/clientpositive/perf/query85.q.out @@ -5,153 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 17 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Map 16 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=1014) - Number of rows:100 - Select Operator [SEL_57] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_54] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=2440165 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col6)","avg(_col16)","avg(_col15)"],keys:_col28 - Select Operator [SEL_51] (rows=2440165 width=1014) - Output:["_col28","_col6","_col16","_col15"] - Merge Join Operator [MERGEJOIN_109] (rows=2440165 width=1014) - Conds:RS_48._col13=RS_49._col0(Inner),Output:["_col6","_col15","_col16","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=72 width=200) - Output:["_col0","_col1"] - Filter Operator [FIL_102] (rows=72 width=200) - predicate:r_reason_sk is not null - TableScan [TS_42] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_108] (rows=2218332 width=1014) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col6","_col13","_col15","_col16"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_95] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=2016666 width=1014) - Output:["_col0","_col11","_col13","_col14","_col4"] - Filter Operator [FIL_40] (rows=2016666 width=1014) - predicate:(((_col23) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 100 AND 200) or ((_col23) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 300) or ((_col23) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 250)) - Select Operator [SEL_39] (rows=12100000 width=1014) - Output:["_col0","_col4","_col6","_col11","_col13","_col14","_col23"] - Merge Join Operator [MERGEJOIN_107] (rows=12100000 width=1014) - Conds:RS_36._col13, _col20, _col21=RS_37._col0, _col1, _col2(Inner),Output:["_col1","_col3","_col7","_col9","_col14","_col16","_col17"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_32] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_101] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_30] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col13, _col20, _col21 - Merge Join Operator [MERGEJOIN_106] (rows=11000000 width=1014) - Conds:RS_33._col0=RS_34._col9(Inner),Output:["_col1","_col3","_col7","_col9","_col13","_col14","_col16","_col17","_col20","_col21"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col9 - Select Operator [SEL_29] (rows=1774698 width=135) - Output:["_col0","_col10","_col11","_col13","_col14","_col17","_col18","_col4","_col6","_col9"] - Filter Operator [FIL_28] (rows=1774698 width=135) - predicate:(((_col17 = 'M') and (_col18 = '4 yr Degree') and _col5 BETWEEN 100 AND 150) or ((_col17 = 'D') and (_col18 = 'Primary') and _col5 BETWEEN 50 AND 100) or ((_col17 = 'U') and (_col18 = 'Advanced Degree') and _col5 BETWEEN 150 AND 200)) - Select Operator [SEL_27] (rows=21296393 width=135) - Output:["_col0","_col4","_col5","_col6","_col9","_col10","_col11","_col13","_col14","_col17","_col18"] - Merge Join Operator [MERGEJOIN_105] (rows=21296393 width=135) - Conds:RS_24._col9=RS_25._col0(Inner),Output:["_col1","_col5","_col6","_col7","_col10","_col11","_col12","_col14","_col15","_col17","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_100] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_104] (rows=19360357 width=135) - Conds:RS_21._col2, _col4=RS_22._col0, _col5(Inner),Output:["_col1","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col14","_col15"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col5 - Select Operator [SEL_14] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_99] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_order_number is not null and wr_refunded_cdemo_sk is not null and wr_returning_cdemo_sk is not null and wr_refunded_addr_sk is not null and wr_reason_sk is not null) - TableScan [TS_12] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_refunded_cdemo_sk","wr_refunded_addr_sk","wr_returning_cdemo_sk","wr_reason_sk","wr_order_number","wr_fee","wr_refunded_cash"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2, _col4 - Merge Join Operator [MERGEJOIN_103] (rows=17600325 width=135) - Conds:RS_18._col0=RS_19._col2(Inner),Output:["_col1","_col2","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col2 - Select Operator [SEL_11] (rows=16000296 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_98] (rows=16000296 width=135) - predicate:((ws_sales_price BETWEEN 100 AND 150 or ws_sales_price BETWEEN 50 AND 100 or ws_sales_price BETWEEN 150 AND 200) and (ws_net_profit BETWEEN 100 AND 200 or ws_net_profit BETWEEN 150 AND 300 or ws_net_profit BETWEEN 50 AND 250) and ws_order_number is not null and ws_item_sk is not null and ws_web_page_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_order_number","ws_quantity","ws_sales_price","ws_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4602 width=585) - Output:["_col0"] - Filter Operator [FIL_97] (rows=4602 width=585) - predicate:wp_web_page_sk is not null - TableScan [TS_6] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query86.q.out b/ql/src/test/results/clientpositive/perf/query86.q.out index 6377c43..8e10e58 100644 --- a/ql/src/test/results/clientpositive/perf/query86.q.out +++ b/ql/src/test/results/clientpositive/perf/query86.q.out @@ -51,76 +51,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_27] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Select Operator [SEL_24] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_23] (rows=261364852 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_22] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_20] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_19] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_17] (rows=522729705 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_15] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_39] (rows=174243235 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_38] (rows=158402938 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_35] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_net_paid"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_36] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query87.q.out b/ql/src/test/results/clientpositive/perf/query87.q.out index 75634d5..99d9814 100644 --- a/ql/src/test/results/clientpositive/perf/query87.q.out +++ b/ql/src/test/results/clientpositive/perf/query87.q.out @@ -5,181 +5,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 19 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_74] - Group By Operator [GBY_72] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_71] - Group By Operator [GBY_70] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_69] (rows=105414409 width=88) - Filter Operator [FIL_68] (rows=105414409 width=88) - predicate:_col6 is null - Merge Join Operator [MERGEJOIN_112] (rows=210828819 width=88) - Conds:RS_65._col0, _col1, _col2=RS_66._col0, _col1, _col2(Left Outer),Output:["_col6"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_63] (rows=87121617 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_61] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_60] (rows=174243235 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_110] (rows=174243235 width=135) - Conds:RS_57._col1=RS_58._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_104] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_51] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_109] (rows=158402938 width=135) - Conds:RS_54._col0=RS_55._col0(Inner),Output:["_col1","_col3"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_102] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_45] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_103] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_48] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_44] (rows=191662559 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=191662559 width=88) - predicate:_col3 is null - Merge Join Operator [MERGEJOIN_111] (rows=383325119 width=88) - Conds:RS_40._col0, _col1, _col2=RS_41._col0, _col1, _col2(Left Outer),Output:["_col0","_col1","_col2","_col3"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_38] (rows=174233858 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_36] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_35] (rows=348467716 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_108] (rows=348467716 width=135) - Conds:RS_32._col1=RS_33._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_101] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_26] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_107] (rows=316788826 width=135) - Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_99] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_20] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_100] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_106] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_98] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_105] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_customer_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_97] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query88.q.out b/ql/src/test/results/clientpositive/perf/query88.q.out index 7b04e9e..9e997e6 100644 --- a/ql/src/test/results/clientpositive/perf/query88.q.out +++ b/ql/src/test/results/clientpositive/perf/query88.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select * from @@ -186,496 +186,43 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 24 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 25 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 36 <- Map 40 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) -Reducer 37 <- Map 41 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 47 (SIMPLE_EDGE) -Reducer 44 <- Map 48 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 49 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE) -Reducer 46 <- Reducer 45 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 51 <- Map 50 (SIMPLE_EDGE), Map 55 (SIMPLE_EDGE) -Reducer 52 <- Map 56 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 53 <- Map 57 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 54 <- Reducer 53 (SIMPLE_EDGE) -Reducer 59 <- Map 58 (SIMPLE_EDGE), Map 63 (SIMPLE_EDGE) -Reducer 6 <- Reducer 14 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Reducer 54 (SIMPLE_EDGE), Reducer 62 (SIMPLE_EDGE) -Reducer 60 <- Map 64 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 61 <- Map 65 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 62 <- Reducer 61 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE), Vertex 49 (SORT_PARTITION_EDGE) +Vertex 46 <- Vertex 45 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 57 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 54 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 64 (SORT_PARTITION_EDGE) +Vertex 61 <- Vertex 60 (SORT_PARTITION_EDGE), Vertex 65 (SORT_PARTITION_EDGE) +Vertex 62 <- Vertex 61 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_218] - Select Operator [SEL_217] (rows=1 width=65) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_347] (rows=1 width=65) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_209] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_328] (rows=766650239 width=88) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_298] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_327] (rows=696954748 width=88) - Conds:RS_41._col2=RS_42._col0(Inner),Output:["_col1"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_297] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_32] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_326] (rows=633595212 width=88) - Conds:RS_38._col0=RS_39._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_295] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_26] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_296] (rows=14400 width=471) - predicate:((t_hour = 12) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_29] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_210] - Group By Operator [GBY_76] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_75] - Group By Operator [GBY_74] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_331] (rows=766650239 width=88) - Conds:RS_70._col1=RS_71._col0(Inner) - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Select Operator [SEL_63] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_302] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_61] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_330] (rows=696954748 width=88) - Conds:RS_67._col2=RS_68._col0(Inner),Output:["_col1"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_68] - PartitionCols:_col0 - Select Operator [SEL_60] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_301] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_58] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_329] (rows=633595212 width=88) - Conds:RS_64._col0=RS_65._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_299] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_52] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_300] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_55] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_211] - Group By Operator [GBY_102] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_101] - Group By Operator [GBY_100] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_334] (rows=766650239 width=88) - Conds:RS_96._col1=RS_97._col0(Inner) - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_306] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_87] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_333] (rows=696954748 width=88) - Conds:RS_93._col2=RS_94._col0(Inner),Output:["_col1"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_305] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_84] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_332] (rows=633595212 width=88) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col2"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_80] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_303] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_78] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Select Operator [SEL_83] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_304] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_81] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_212] - Group By Operator [GBY_128] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_127] - Group By Operator [GBY_126] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_337] (rows=766650239 width=88) - Conds:RS_122._col1=RS_123._col0(Inner) - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Select Operator [SEL_115] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_310] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_113] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88) - Conds:RS_119._col2=RS_120._col0(Inner),Output:["_col1"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_120] - PartitionCols:_col0 - Select Operator [SEL_112] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_309] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_110] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88) - Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_106] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_307] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_104] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Select Operator [SEL_109] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_308] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_107] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_213] - Group By Operator [GBY_154] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_153] - Group By Operator [GBY_152] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_340] (rows=766650239 width=88) - Conds:RS_148._col1=RS_149._col0(Inner) - <-Map 49 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_314] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_139] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_339] (rows=696954748 width=88) - Conds:RS_145._col2=RS_146._col0(Inner),Output:["_col1"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_146] - PartitionCols:_col0 - Select Operator [SEL_138] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_313] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_136] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_145] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_338] (rows=633595212 width=88) - Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col1","_col2"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_142] - PartitionCols:_col0 - Select Operator [SEL_132] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_311] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_130] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_312] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_133] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_208] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_325] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_294] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_293] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_291] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_292] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 54 [SIMPLE_EDGE] - SHUFFLE [RS_214] - Group By Operator [GBY_180] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_179] - Group By Operator [GBY_178] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_343] (rows=766650239 width=88) - Conds:RS_174._col1=RS_175._col0(Inner) - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col0 - Select Operator [SEL_167] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_318] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_165] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_174] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_342] (rows=696954748 width=88) - Conds:RS_171._col2=RS_172._col0(Inner),Output:["_col1"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Select Operator [SEL_164] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_317] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_162] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_341] (rows=633595212 width=88) - Conds:RS_168._col0=RS_169._col0(Inner),Output:["_col1","_col2"] - <-Map 50 [SIMPLE_EDGE] - SHUFFLE [RS_168] - PartitionCols:_col0 - Select Operator [SEL_158] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_315] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_156] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col0 - Select Operator [SEL_161] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_316] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_159] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 62 [SIMPLE_EDGE] - SHUFFLE [RS_215] - Group By Operator [GBY_206] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 61 [SIMPLE_EDGE] - SHUFFLE [RS_205] - Group By Operator [GBY_204] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_346] (rows=766650239 width=88) - Conds:RS_200._col1=RS_201._col0(Inner) - <-Map 65 [SIMPLE_EDGE] - SHUFFLE [RS_201] - PartitionCols:_col0 - Select Operator [SEL_193] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_322] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_191] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_200] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_345] (rows=696954748 width=88) - Conds:RS_197._col2=RS_198._col0(Inner),Output:["_col1"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_198] - PartitionCols:_col0 - Select Operator [SEL_190] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_321] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_188] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_197] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_344] (rows=633595212 width=88) - Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1","_col2"] - <-Map 58 [SIMPLE_EDGE] - SHUFFLE [RS_194] - PartitionCols:_col0 - Select Operator [SEL_184] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_319] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_182] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_195] - PartitionCols:_col0 - Select Operator [SEL_187] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_320] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_185] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query89.q.out b/ql/src/test/results/clientpositive/perf/query89.q.out index 7bc8700..4ec3f63 100644 --- a/ql/src/test/results/clientpositive/perf/query89.q.out +++ b/ql/src/test/results/clientpositive/perf/query89.q.out @@ -55,95 +55,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_30] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_46] (rows=191662559 width=88) - predicate:CASE WHEN ((avg_window_0 <> 0)) THEN (((abs((_col6 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END - Select Operator [SEL_29] (rows=383325119 width=88) - Output:["avg_window_0","_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_28] (rows=383325119 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col2, _col3, _col4"}] - Select Operator [SEL_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col2, _col3, _col4 - Select Operator [SEL_25] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_24] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_22] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col7, _col10, _col12, _col13 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col5","_col6","_col7","_col10","_col12","_col13","_col3"] - Merge Join Operator [MERGEJOIN_53] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col7","_col10","_col12","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=696954748 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36525 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_49] (rows=36525 width=1119) - predicate:((d_year) IN (2000) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_51] (rows=633595212 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_48] (rows=231000 width=1436) - predicate:(((i_class) IN ('wallpaper', 'parenting', 'musical') or (i_class) IN ('womens', 'birdal', 'pants')) and ((i_category) IN ('Home', 'Books', 'Electronics') or (i_category) IN ('Shoes', 'Jewelry', 'Men')) and (((i_category) IN ('Home', 'Books', 'Electronics') and (i_class) IN ('wallpaper', 'parenting', 'musical')) or ((i_category) IN ('Shoes', 'Jewelry', 'Men') and (i_class) IN ('womens', 'birdal', 'pants'))) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query90.q.out b/ql/src/test/results/clientpositive/perf/query90.q.out index 92eeba1..aa06c30 100644 --- a/ql/src/test/results/clientpositive/perf/query90.q.out +++ b/ql/src/test/results/clientpositive/perf/query90.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 @@ -6,143 +6,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_59] - Limit [LIM_58] (rows=1 width=17) - Number of rows:100 - Select Operator [SEL_57] (rows=1 width=17) - Output:["_col0"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1 width=17) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_92] (rows=1 width=17) - Conds:(Inner),Output:["_col0","_col1"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_53] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_85] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_84] (rows=9600 width=471) - predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) - TableScan [TS_32] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_28] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_26] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_83] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_29] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_52] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_81] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_80] (rows=9600 width=471) - predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) - TableScan [TS_6] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_79] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_3] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query91.q.out b/ql/src/test/results/clientpositive/perf/query91.q.out index 1538e8b..fda2585 100644 --- a/ql/src/test/results/clientpositive/perf/query91.q.out +++ b/ql/src/test/results/clientpositive/perf/query91.q.out @@ -5,126 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_47] - Select Operator [SEL_46] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_45] - Select Operator [SEL_44] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_43] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_41] (rows=117128008 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col8, _col9, _col10, _col18, _col19 - Select Operator [SEL_40] (rows=117128008 width=860) - Output:["_col8","_col9","_col10","_col18","_col19","_col3"] - Merge Join Operator [MERGEJOIN_81] (rows=117128008 width=860) - Conds:RS_37._col1=RS_38._col2(Inner),Output:["_col3","_col8","_col9","_col10","_col18","_col19"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_30] (rows=106480005 width=860) - Output:["_col2","_col7","_col8"] - Merge Join Operator [MERGEJOIN_80] (rows=106480005 width=860) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col0","_col5","_col6"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_75] (rows=3600 width=107) - predicate:((hd_buy_potential like '0-500%') and hd_demo_sk is not null) - TableScan [TS_18] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=96800003 width=860) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col2","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_74] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) - TableScan [TS_15] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_78] (rows=88000001 width=860) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=930900 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_73] (rows=930900 width=385) - predicate:(((cd_education_status = 'Unknown') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'W')) and (((cd_marital_status = 'M') and (cd_education_status = 'Unknown')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree'))) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_9] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_77] (rows=34846646 width=106) - Conds:RS_34._col2=RS_35._col0(Inner),Output:["_col1","_col3","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=60 width=2045) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_71] (rows=60 width=2045) - predicate:cc_call_center_sk is not null - TableScan [TS_6] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_call_center_id","cc_name","cc_manager"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_76] (rows=31678769 width=106) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_69] (rows=28798881 width=106) - predicate:(cr_call_center_sk is not null and cr_returned_date_sk is not null and cr_returning_customer_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_call_center_sk","cr_net_loss"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_70] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query92.q.out b/ql/src/test/results/clientpositive/perf/query92.q.out index 28df749..74fedcc 100644 --- a/ql/src/test/results/clientpositive/perf/query92.q.out +++ b/ql/src/test/results/clientpositive/perf/query92.q.out @@ -5,89 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_37] - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_48] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_46] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,ss,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_43] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_47] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_45] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_44] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query93.q.out b/ql/src/test/results/clientpositive/perf/query93.q.out index f28ba41..57ea1e0 100644 --- a/ql/src/test/results/clientpositive/perf/query93.q.out +++ b/ql/src/test/results/clientpositive/perf/query93.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=316797606 width=88) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_15] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_34] (rows=633595212 width=88) - Conds:RS_12._col0, _col2=RS_13._col0, _col2(Inner),Output:["_col3","_col7","_col9","_col10"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0, _col2 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_32] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_customer_sk","ss_ticket_number","ss_quantity","ss_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0, _col2 - Merge Join Operator [MERGEJOIN_33] (rows=63350266 width=77) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_30] (rows=57591150 width=77) - predicate:(sr_reason_sk is not null and sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_reason_sk","sr_ticket_number","sr_return_quantity"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36 width=200) - Output:["_col0"] - Filter Operator [FIL_31] (rows=36 width=200) - predicate:((r_reason_desc = 'Did not like the warranty') and r_reason_sk is not null) - TableScan [TS_3] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query94.q.out b/ql/src/test/results/clientpositive/perf/query94.q.out index 836b16b..43b7f98 100644 --- a/ql/src/test/results/clientpositive/perf/query94.q.out +++ b/ql/src/test/results/clientpositive/perf/query94.q.out @@ -5,126 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 14 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=344) - Number of rows:100 - Group By Operator [GBY_48] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Group By Operator [GBY_46] (rows=127554770 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Select Operator [SEL_45] (rows=127554770 width=135) - Output:["_col3","_col4","_col5"] - Filter Operator [FIL_44] (rows=127554770 width=135) - predicate:_col12 is null - Merge Join Operator [MERGEJOIN_85] (rows=255109540 width=135) - Conds:RS_40._col3=RS_41._col0(Left Outer),Output:["_col3","_col4","_col5","_col12"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_79] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_23] (rows=14398467 width=92) - default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=231917759 width=135) - Conds:RS_37._col2=RS_38._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_78] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_20] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=210834322 width=135) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_77] (rows=20000000 width=1014) - predicate:((ca_state = 'TX') and ca_address_sk is not null) - TableScan [TS_17] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_82] (rows=191667562 width=135) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_date BETWEEN '1999-05-01' AND '1999-07-01' and d_date_sk is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=174243235 width=135) - Conds:RS_28._col3=RS_29._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_73] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_80] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 7 diff --git a/ql/src/test/results/clientpositive/perf/query95.q.out b/ql/src/test/results/clientpositive/perf/query95.q.out index 9b0d1b2..7812cc2 100644 --- a/ql/src/test/results/clientpositive/perf/query95.q.out +++ b/ql/src/test/results/clientpositive/perf/query95.q.out @@ -5,152 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_63] - Group By Operator [GBY_61] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Group By Operator [GBY_59] (rows=510219083 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Merge Join Operator [MERGEJOIN_122] (rows=510219083 width=135) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_115] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_38] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_121] (rows=463835520 width=135) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_114] (rows=20000000 width=1014) - predicate:((ca_state = 'GA') and ca_address_sk is not null) - TableScan [TS_35] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_120] (rows=421668646 width=135) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_113] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2002-05-01' AND '2002-06-30' and d_date_sk is not null) - TableScan [TS_32] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=383335125 width=135) - Conds:RS_45._col3=RS_46._col0(Left Semi),RS_45._col3=RS_47._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_44] (rows=174243235 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_31] (rows=174243235 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_118] (rows=174243235 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_112] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_25] (rows=14398467 width=92) - default@web_returns,wr,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_23] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_117] (rows=158402938 width=135) - Conds:RS_20._col1=RS_21._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_16] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_14] (rows=144002668 width=135) - default@web_sales,ws4,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_111] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_17] (rows=144002668 width=135) - default@web_sales,ws5,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Group By Operator [GBY_42] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_116] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query96.q.out b/ql/src/test/results/clientpositive/perf/query96.q.out index 8ffcbcb..349c39a 100644 --- a/ql/src/test/results/clientpositive/perf/query96.q.out +++ b/ql/src/test/results/clientpositive/perf/query96.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=1 width=8) - Number of rows:100 - Select Operator [SEL_27] (rows=1 width=8) - Output:["_col0"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_45] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_42] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=696954748 width=88) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_41] (rows=3600 width=107) - predicate:((hd_dep_count = 5) and hd_demo_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_40] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/perf/query97.q.out b/ql/src/test/results/clientpositive/perf/query97.q.out index c885e99..d1603e7 100644 --- a/ql/src/test/results/clientpositive/perf/query97.q.out +++ b/ql/src/test/results/clientpositive/perf/query97.q.out @@ -5,91 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Limit [LIM_37] (rows=1 width=24) - Number of rows:100 - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_49] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_47] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_48] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_45] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git a/ql/src/test/results/clientpositive/perf/query98.q.out b/ql/src/test/results/clientpositive/perf/query98.q.out index 35cc897..a4af685 100644 --- a/ql/src/test/results/clientpositive/perf/query98.q.out +++ b/ql/src/test/results/clientpositive/perf/query98.q.out @@ -5,74 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_27] - Select Operator [SEL_26] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_37] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_35] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_36] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_33] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_34] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2001-01-12 AND 2001-02-11 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index 05a4f5f..3fb3c09 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -13,30 +13,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:10 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_1] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/13 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_3] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/13 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select key from src PREHOOK: type: QUERY @@ -84,13 +70,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -128,13 +108,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 PREHOOK: query: select key from src limit 10 PREHOOK: type: QUERY @@ -232,23 +206,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=205/309 width=95) - Output:["_col0","_col1"],aggregations:["count(KEY._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - PartitionCols:key - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["key"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select key, count(key) from src group by key)subq PREHOOK: type: QUERY @@ -274,39 +238,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_18] (rows=382/0 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=166/497 width=87) - Output:["_col0"] - Filter Operator [FIL_16] (rows=166/497 width=87) - predicate:(UDFToDouble(key) > 0.0) - TableScan [TS_0] (rows=500/500 width=87) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=166/0 width=91) - Output:["_col0"] - Filter Operator [FIL_17] (rows=166/0 width=91) - predicate:(UDFToDouble(value) > 0.0) - TableScan [TS_3] (rows=500/500 width=91) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] + Vertex 3 PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0 PREHOOK: type: QUERY @@ -323,39 +262,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_18] (rows=399/1019 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=166/497 width=87) - Output:["_col0"] - Filter Operator [FIL_16] (rows=166/497 width=87) - predicate:(UDFToDouble(key) > 0.0) - TableScan [TS_0] (rows=500/500 width=87) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=166/497 width=87) - Output:["_col0"] - Filter Operator [FIL_17] (rows=166/497 width=87) - predicate:(UDFToDouble(key) > 0.0) - TableScan [TS_3] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 3 PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0 PREHOOK: type: QUERY @@ -381,26 +295,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_3] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/500 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select count(*) from (select * from src a union all select * from src b)subq PREHOOK: type: QUERY @@ -432,34 +336,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=1219/1028 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_15] (rows=1219/1028 width=178) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=87) - default@src,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 641daf5..de215f6 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -41,139 +41,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_56] - Group By Operator [GBY_54] (rows=28/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_49] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_82] (rows=148/61 width=177) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_78] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_40] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_81] (rows=61/52 width=177) - Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_37] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 16 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_80] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_74] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=61/52 width=177) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_73] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_71] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 8 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_72] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 7 PREHOOK: query: SELECT x.key, y.value FROM src1 x JOIN src y ON (x.key = y.key) @@ -234,264 +118,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_114] - Group By Operator [GBY_112] (rows=872/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=434/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_162] (rows=434/61 width=177) - Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_98] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_161] (rows=178/52 width=86) - Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_97] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_95] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Select Operator [SEL_94] (rows=1525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_93] (rows=1525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_88] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_86] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_85] (rows=1025/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_84] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_79] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_77] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_76] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_75] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_67] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_65] (rows=25/25 width=175) - Output:["key","value"] - <-Map 33 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_70] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_68] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Group By Operator [GBY_63] (rows=438/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=290/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_160] (rows=290/61 width=177) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_49] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_159] (rows=119/52 width=86) - Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_46] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=1025/319 width=178) - Output:["_col1"] - Group By Operator [GBY_44] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_39] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_37] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_146] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 20 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_147] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_158] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_157] (rows=61/52 width=86) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_144] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_142] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 10 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_143] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 9 PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -647,22 +302,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -683,42 +323,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key PREHOOK: type: QUERY @@ -746,22 +357,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -786,42 +382,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab2,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key UNION ALL @@ -858,59 +425,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_25] - Group By Operator [GBY_23] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_22] - Merge Join Operator [MERGEJOIN_38] (rows=558/1646 width=18) - Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=18) - Output:["_col0"] - Filter Operator [FIL_35] (rows=500/500 width=18) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=18) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_36] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_33] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_32] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - Output:["key"] - <-Map 6 [CONTAINS] - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_34] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_10] (rows=242/242 width=18) - Output:["key"] + Vertex 4 PREHOOK: query: select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value UNION ALL @@ -947,73 +471,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_31] - Group By Operator [GBY_29] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Merge Join Operator [MERGEJOIN_50] (rows=587/3768 width=18) - Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500/500 width=18) - Output:["_col0"] - Filter Operator [FIL_46] (rows=500/500 width=18) - predicate:key is not null - TableScan [TS_21] (rows=500/500 width=18) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_45] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_16] (rows=242/242 width=18) - Output:["key"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_49] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_47] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_43] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_42] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_44] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 5 PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -1126,6 +593,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 13 <- Union 10 (CONTAINS) Map 14 <- Union 10 (CONTAINS) @@ -1141,6 +609,23 @@ Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 22 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 13 <- Union 10 (CONTAINS) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 17 <- Union 18 (CONTAINS) +Vertex 19 <- Union 18 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 18 (CONTAINS) +Vertex 21 <- Union 18 (CONTAINS) +Vertex 22 <- Union 18 (CONTAINS) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 10 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1150,6 +635,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Union 5 <-Reducer 12 [CONTAINS] File Output Operator [FS_79] @@ -1343,6 +829,12 @@ Stage-5 File Output Operator [FS_83] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_20] +======= + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 3 [CONTAINS] +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1431,6 +923,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 10 <- Union 2 (CONTAINS) Map 13 <- Union 14 (CONTAINS) @@ -1454,6 +947,31 @@ Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 14 (CONTAINS) +Vertex 20 <- Union 16 (CONTAINS) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 7 (CONTAINS), Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 25 (CONTAINS) +Vertex 33 <- Union 27 (CONTAINS) +Vertex 34 <- Union 29 (CONTAINS) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1463,6 +981,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Reducer 9 File Output Operator [FS_115] table:{"name:":"default.a"} @@ -1700,6 +1219,9 @@ Stage-5 File Output Operator [FS_119] table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_112] +======= + Vertex 8 +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1764,10 +1286,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -1777,6 +1299,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 5 File Output Operator [FS_18] table:{"name:":"default.dest1"} @@ -1812,6 +1335,9 @@ Stage-4 Group By Operator [GBY_21] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_12] +======= + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -1846,33 +1372,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=594/4122 width=260) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_9] (rows=594/4122 width=260) - Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=500/500 width=87) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:key - TableScan [TS_1] (rows=25/25 width=86) - default@src1,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=2000/2000 width=87) - default@srcpart,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: SELECT TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -1905,32 +1411,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=1219/1028 width=178) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=1219/1028 width=178) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_1] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: FROM ( select key, value from ( @@ -1991,11 +1478,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Map 7 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) +Vertex 7 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -2005,6 +1492,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_17] table:{"name:":"default.dest1"} @@ -2052,6 +1540,10 @@ Stage-4 Group By Operator [GBY_20] (rows=1001/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -2106,10 +1598,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -2119,6 +1611,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_14] table:{"name:":"default.dest1"} @@ -2156,6 +1649,10 @@ Stage-4 Group By Operator [GBY_17] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 8d76ad2..d2694e4 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -227,9 +227,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -244,21 +242,12 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics for columns POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-2 Column Stats Work{} Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=10) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) PREHOOK: type: CREATEMACRO @@ -345,13 +334,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -599,24 +582,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Limit [LIM_4] (rows=5/5 width=178) - Number of rows:5 - Select Operator [SEL_3] (rows=500/5 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -660,15 +632,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 - File Output Operator [FS_3] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_2] (rows=306/3 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_4] (rows=306/3 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919/15000 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) @@ -823,31 +787,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (CUSTOM_EDGE) +Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 2 - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=550/480 width=18) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_15] (rows=550/480 width=18) - BucketMapJoin:true,Conds:RS_6._col0=SEL_5._col0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col3"] - <-Map 1 [CUSTOM_EDGE] - MULTICAST [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_5] (rows=500/500 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500/500 width=18) - predicate:key is not null - TableScan [TS_3] (rows=500/500 width=18) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out index 27c1bbe..d348e90 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out @@ -37,39 +37,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166/10 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166/10 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019/10 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -132,39 +107,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_18] (rows=2166/10 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365/10 width=2) - Output:["_col0"] - Filter Operator [FIL_16] (rows=1365/10 width=2) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=2) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019/10 width=8) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1019/10 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 3 PREHOOK: query: select count(*) @@ -224,45 +174,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=615/5 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=615/5 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_20] (rows=2166/10 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019/10 width=8) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1019/10 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -328,39 +248,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -423,39 +318,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1/1 width=8) - Output:["_col0"],aggregations:["count()"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_18] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true - <-Map 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0"] - Filter Operator [FIL_16] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 3 PREHOOK: query: select count(*) @@ -515,45 +385,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=750/5 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=750/5 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Map Join Operator [MAPJOIN_20] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index b9d69fd..3df6949 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -29,9 +29,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -46,21 +44,12 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics for columns POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-2 Column Stats Work{} Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=10) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table src_multi2 PREHOOK: type: DROPTABLE @@ -91,11 +80,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) Stage-4 Column Stats Work{} @@ -107,56 +96,7 @@ Stage-4 Stage-2 Dependency Collection{} Stage-1 - Reducer 5 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] - <-Reducer 4 [SIMPLE_EDGE] - File Output Operator [FS_19] - table:{"name:":"default.src_multi2"} - Select Operator [SEL_18] (rows=1280/508 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_26] (rows=1280/508 width=178) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_12] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col0"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_2] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 6 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=25/25 width=175) - predicate:key is not null - TableScan [TS_3] (rows=25/25 width=175) - Output:["key","value"] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=1280/508 width=178) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_18] + Vertex 5 PREHOOK: query: select count(*) from (select * from src union select * from src1)subq PREHOOK: type: QUERY @@ -283,7 +223,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -293,20 +233,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_8] - table:{"name:":"default.acid_uami"} - Select Operator [SEL_4] (rows=8/2 width=302) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=8/2 width=302) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_9] (rows=8/2 width=226) - predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=8/4 width=226) - default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"] + Vertex 2 PREHOOK: query: select * from acid_uami order by de PREHOOK: type: QUERY @@ -396,7 +323,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze delete from acid_dot where cint < -1070551679 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -406,20 +333,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_7] - table:{"name:":"default.acid_dot"} - Select Operator [SEL_4] (rows=31436/8 width=4) - Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=31436/8 width=4) - Output:["_col0"] - Filter Operator [FIL_8] (rows=31436/8 width=4) - predicate:(cint < -1070551679) - TableScan [TS_0] (rows=94309/12288 width=4) - default@acid_dot,acid_dot, ACID table,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 2 PREHOOK: query: select count(*) from acid_dot PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 5ba02f3..763507c 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -23,22 +23,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=16 width=107) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_6] - Select Operator [SEL_5] (rows=16 width=107) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=16 width=107) - default@acid_vectorized,acid_vectorized, ACID table,Tbl:COMPLETE,Col:NONE,Output:["a","b"] + Vertex 2 PREHOOK: query: explain select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol @@ -181,32 +172,19 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: explain analyze table src compute statistics for columns PREHOOK: type: QUERY POSTHOOK: query: explain analyze table src compute statistics for columns POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-2 Column Stats Work{} Stage-0 - Reducer 2 - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=984) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"] - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) @@ -263,13 +241,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -443,24 +415,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 vectorized - File Output Operator [FS_8] - Limit [LIM_7] (rows=5 width=178) - Number of rows:5 - Select Operator [SEL_6] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -496,15 +457,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 vectorized - File Output Operator [FS_10] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_9] (rows=306 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_8] (rows=306 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) @@ -643,31 +596,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (CUSTOM_EDGE) +Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 2 - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=550 width=18) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_15] (rows=550 width=18) - BucketMapJoin:true,Conds:RS_6._col0=SEL_5._col0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col3"] - <-Map 1 [CUSTOM_EDGE] - MULTICAST [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242 width=18) - default@tab,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_5] (rows=500 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500 width=18) - predicate:key is not null - TableScan [TS_3] (rows=500 width=18) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index 63d8546..1e995c0 100644 --- a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -48,11 +48,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -71,7 +71,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -82,7 +82,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -99,20 +113,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -168,11 +168,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -191,7 +191,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -203,7 +203,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -220,20 +234,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -285,11 +285,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -308,7 +308,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -319,7 +319,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -336,20 +350,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -401,11 +401,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -424,7 +424,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -436,7 +436,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -453,20 +467,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -516,11 +516,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -536,7 +536,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -547,21 +547,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -575,6 +561,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -624,11 +624,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -644,7 +644,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -656,21 +656,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -684,6 +670,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -767,11 +767,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -790,7 +790,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -801,7 +801,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -818,20 +832,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -885,11 +885,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -908,7 +908,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -920,7 +920,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -937,20 +951,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1004,11 +1004,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1027,7 +1027,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1038,7 +1038,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1055,20 +1069,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1122,11 +1122,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1145,7 +1145,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1157,7 +1157,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1174,20 +1188,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1250,7 +1250,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1270,10 +1270,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1293,7 +1293,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1307,7 +1307,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1331,7 +1331,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1446,7 +1446,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1466,10 +1466,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1489,7 +1489,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1503,7 +1503,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1527,7 +1527,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out index 473daca..4bf3b55 100644 --- a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -58,7 +58,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -75,8 +75,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -87,20 +87,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -114,6 +101,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -165,11 +165,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -182,7 +182,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -199,8 +199,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -212,20 +212,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -239,6 +226,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -303,11 +303,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -320,7 +320,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -339,9 +339,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -352,7 +352,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -365,7 +379,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -378,20 +392,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -447,11 +447,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -464,7 +464,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -483,9 +483,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -497,7 +497,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -510,7 +524,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -523,20 +537,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -609,14 +609,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -629,7 +631,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -642,7 +644,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -659,8 +661,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -671,7 +673,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -684,7 +717,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -697,7 +730,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -714,8 +747,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -726,38 +759,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -774,8 +776,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -844,14 +844,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -864,7 +866,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -877,7 +879,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -894,8 +896,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -907,7 +909,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -920,7 +953,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -933,7 +966,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -950,8 +983,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -963,38 +996,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1011,8 +1013,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1094,11 +1094,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1112,7 +1112,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1130,8 +1130,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1142,8 +1142,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1154,7 +1154,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1167,7 +1181,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1180,7 +1194,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1193,20 +1207,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1278,11 +1278,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1296,7 +1296,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1314,8 +1314,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Map Join Operator @@ -1327,8 +1327,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -1340,7 +1340,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1353,7 +1367,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1366,7 +1380,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1379,20 +1393,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/multi_count_distinct.q.out b/ql/src/test/results/clientpositive/tez/multi_count_distinct.q.out index 744c6d2..8ef11c7 100644 --- a/ql/src/test/results/clientpositive/tez/multi_count_distinct.q.out +++ b/ql/src/test/results/clientpositive/tez/multi_count_distinct.q.out @@ -26,34 +26,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Group By Operator [GBY_10] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_9] - Group By Operator [GBY_8] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","count(_col1)","count(_col2)"] - Select Operator [SEL_6] (rows=13 width=5) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=13 width=5) - Output:["_col0","_col1","_col2","_col3"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_3] (rows=27 width=5) - Output:["_col0","_col1","_col2","_col3"],keys:_col0, _col1, _col2, 0 - Select Operator [SEL_1] (rows=9 width=5) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=9 width=5) - default@employee,employee,Tbl:COMPLETE,Col:NONE,Output:["department_id","gender","education_level"] + Vertex 3 PREHOOK: query: select count(distinct department_id), count(distinct gender), count(distinct education_level) from employee PREHOOK: type: QUERY @@ -104,36 +84,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1 width=40) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_10] (rows=1 width=40) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","count(VALUE._col4)"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_9] - Group By Operator [GBY_8] (rows=1 width=40) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(_col0)","count(_col1)","count(_col2)","count(_col3)","count(_col4)"] - Select Operator [SEL_6] (rows=22 width=5) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_5] (rows=22 width=5) - Output:["_col0","_col1","_col2","_col3"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_3] (rows=45 width=5) - Output:["_col0","_col1","_col2","_col3"],keys:_col0, _col1, _col2, 0 - Select Operator [SEL_1] (rows=9 width=5) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=9 width=5) - default@employee,employee,Tbl:COMPLETE,Col:NONE,Output:["gender","department_id","education_level"] + Vertex 3 PREHOOK: query: select count(distinct gender), count(distinct department_id), count(distinct gender), count(distinct education_level), count(distinct education_level, department_id), count(distinct department_id, education_level), count(distinct department_id, education_level, gender) from employee diff --git a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out index 8006a90..4165c57 100644 --- a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out @@ -102,36 +102,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_21] (rows=2 width=431) - Conds:RS_23._col2=RS_28._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_23] - PartitionCols:_col2 - Select Operator [SEL_22] (rows=2 width=134) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=2 width=236) - default@char_tbl1,c1,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] - Dynamic Partitioning Event Operator [EVENT_26] (rows=1 width=134) - Group By Operator [GBY_25] (rows=1 width=134) - Output:["_col0"],keys:_col0 - Select Operator [SEL_24] (rows=2 width=134) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_22] - <-Map 3 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_28] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=2 width=89) - Output:["_col0","_col1","_col2"] - TableScan [TS_3] (rows=2 width=190) - default@char_tbl2,c2,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] + Vertex 2 PREHOOK: query: select c1.name, c1.age, c1.gpa, c2.name, c2.age, c2.gpa from char_tbl1 c1 join char_tbl2 c2 on (c1.gpa = c2.gpa) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out b/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out index 4535e66..624ace1 100644 --- a/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out @@ -40,10 +40,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -62,7 +62,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: tinyint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -119,10 +119,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -140,7 +140,7 @@ STAGE PLANS: Statistics: Num rows: 1024 Data size: 121205 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out index 12b90a4..7769b42 100644 --- a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out @@ -154,10 +154,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -176,7 +176,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: decimal(22,21)) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -331,10 +331,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -353,7 +353,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out index 71e470b..c4a7073 100644 --- a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -79,7 +79,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col2 (type: smallint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -148,10 +148,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -174,7 +174,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: struct) Execution mode: vectorized - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0) @@ -244,10 +244,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -268,7 +268,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -336,10 +336,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -359,7 +359,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -461,11 +461,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -490,7 +490,7 @@ STAGE PLANS: Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -504,7 +504,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: vectorized Reduce Operator Tree: Select Operator