diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java index f00fc77..bd10912 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java @@ -277,4 +277,25 @@ public static boolean sameRowSchema(Operator operator1, Operator operator2 } return resultMap.build(); } + + /** + * Given an operator and a set of classes, it returns the number of operators it finds + * upstream that instantiate any of the given classes. + * + * @param start the start operator + * @param classes the set of classes + * @return the number of operators + */ + public static int countOperatorsUpstream(Operator start, Set>> classes) { + Multimap>, Operator> ops = classifyOperatorsUpstream(start, classes); + int numberOperators = 0; + Set> uniqueOperators = new HashSet>(); + for (Operator op : ops.values()) { + if (uniqueOperators.add(op)) { + numberOperators++; + } + } + return numberOperators; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 024849e..f2ffb3c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -32,18 +32,22 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.MuxOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.OperatorUtils; +import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.TezDummyStoreOperator; +import org.apache.hadoop.hive.ql.exec.UDTFOperator; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; @@ -61,6 +65,8 @@ import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.util.ReflectionUtils; +import com.google.common.collect.ImmutableSet; + /** * ConvertJoinMapJoin is an optimization that replaces a common join * (aka shuffle join) with a map join (aka broadcast or fragment replicate @@ -70,7 +76,18 @@ */ public class ConvertJoinMapJoin implements NodeProcessor { - static final private Log LOG = LogFactory.getLog(ConvertJoinMapJoin.class.getName()); + private static final Log LOG = LogFactory.getLog(ConvertJoinMapJoin.class.getName()); + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static final Set>> COSTLY_OPERATORS = + new ImmutableSet.Builder() + .add(CommonJoinOperator.class) + .add(GroupByOperator.class) + .add(LateralViewJoinOperator.class) + .add(PTFOperator.class) + .add(ReduceSinkOperator.class) + .add(UDTFOperator.class) + .build(); @Override /* @@ -538,16 +555,20 @@ public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext c HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD); int bigTablePosition = -1; - + // number of costly ops (Join, GB, PTF/Windowing, TF) below the big input + int bigInputNumberCostlyOps = -1; + // stats of the big input Statistics bigInputStat = null; - long totalSize = 0; - int pos = 0; // bigTableFound means we've encountered a table that's bigger than the // max. This table is either the the big table or we cannot convert. boolean bigTableFound = false; - for (Operator parentOp : joinOp.getParentOperators()) { + // total size of the inputs + long totalSize = 0; + + for (int pos = 0; pos < joinOp.getParentOperators().size(); pos++) { + Operator parentOp = joinOp.getParentOperators().get(pos); Statistics currInputStat = parentOp.getStatistics(); if (currInputStat == null) { @@ -556,15 +577,17 @@ public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext c } long inputSize = currInputStat.getDataSize(); + + boolean currentBigTable = false; if ((bigInputStat == null) - || ((bigInputStat != null) && (inputSize > bigInputStat.getDataSize()))) { + || ((bigInputStat != null) && (inputSize > bigInputStat.getDataSize()))) { if (bigTableFound) { // cannot convert to map join; we've already chosen a big table // on size and there's another one that's bigger. return -1; } - + if (inputSize/buckets > maxSize) { if (!bigTableCandidateSet.contains(pos)) { // can't use the current table as the big table, but it's too @@ -572,33 +595,44 @@ public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext c return -1; } + currentBigTable = true; bigTableFound = true; } + } - if (bigInputStat != null) { - // we're replacing the current big table with a new one. Need - // to count the current one as a map table then. - totalSize += bigInputStat.getDataSize(); - } + int inputNumberCostlyOps = OperatorUtils.countOperatorsUpstream(parentOp, COSTLY_OPERATORS); + + // This input is the big table if it is contained in the big candidates set, and either: + // 1) we have not chosen a big table yet, or + // 2) it has been chosen as the big table above, or + // 3) the number of costly operators for this input is higher, or + // 4) the number of costly operators is equal, but the size is bigger, + boolean selectedBigTable = bigTableCandidateSet.contains(pos) && + (bigInputStat == null || currentBigTable || inputNumberCostlyOps > bigInputNumberCostlyOps || + (inputNumberCostlyOps == bigInputNumberCostlyOps && inputSize > bigInputStat.getDataSize())); + + if (bigInputStat != null && selectedBigTable) { + // We are replacing the current big table with a new one, thus + // we need to count the current one as a map table then. + totalSize += bigInputStat.getDataSize(); + } else if (!selectedBigTable) { + // This is not the first table and we are not using it as big table, + // in fact, we're adding this table as a map table + totalSize += inputSize; + } - if (totalSize/buckets > maxSize) { - // sum of small tables size in this join exceeds configured limit - // hence cannot convert. - return -1; - } + if (totalSize/buckets > maxSize) { + // sum of small tables size in this join exceeds configured limit + // hence cannot convert. + return -1; + } - if (bigTableCandidateSet.contains(pos)) { - bigTablePosition = pos; - bigInputStat = currInputStat; - } - } else { - totalSize += currInputStat.getDataSize(); - if (totalSize/buckets > maxSize) { - // cannot hold all map tables in memory. Cannot convert. - return -1; - } + if (selectedBigTable) { + bigTablePosition = pos; + bigInputNumberCostlyOps = inputNumberCostlyOps; + bigInputStat = currInputStat; } - pos++; + } return bigTablePosition; @@ -616,7 +650,6 @@ public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext c * * for tez. */ - public MapJoinOperator convertJoinMapJoin(JoinOperator joinOp, OptimizeTezProcContext context, int bigTablePosition, boolean removeReduceSink) throws SemanticException { // bail on mux operator because currently the mux operator masks the emit keys diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 4699e10..1f1bf3d 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -329,8 +329,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -362,31 +361,15 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -398,12 +381,28 @@ STAGE PLANS: expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -556,8 +555,7 @@ STAGE PLANS: Tez Edges: Map 1 <- Map 3 (CUSTOM_EDGE) - Map 4 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -614,31 +612,15 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -650,12 +632,28 @@ STAGE PLANS: expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -871,8 +869,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -904,31 +901,15 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -936,12 +917,28 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -967,8 +964,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -994,31 +990,15 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1026,12 +1006,28 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out index 111aaaa..14da817 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out @@ -456,7 +456,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Map 1 <- Map 2 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -471,24 +471,6 @@ STAGE PLANS: expressions: key (type: int) outputColumnNames: _col0 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - Map 2 - Map Operator Tree: - TableScan - alias: tab_part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 2) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Right Outer Join0 to 1 @@ -497,7 +479,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 1 Map 2 Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true File Output Operator @@ -508,6 +490,24 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap + Map 2 + Map Operator Tree: + TableScan + alias: tab_part + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap Stage: Stage-0 Fetch Operator @@ -527,8 +527,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -558,38 +557,38 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -609,8 +608,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -640,43 +638,43 @@ STAGE PLANS: Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col2 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Execution mode: llap Reducer 2 - Execution mode: llap + Execution mode: uber Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col2 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 3ebd690..d5e2c7c 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -4262,7 +4262,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4277,9 +4277,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4290,26 +4289,11 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart @@ -4330,7 +4314,35 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 2 + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: uber Reduce Operator Tree: Group By Operator @@ -4345,19 +4357,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -4365,7 +4364,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4567,8 +4566,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4576,13 +4575,33 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 ds (type: string) + 1 ds (type: string) + outputColumnNames: _col8 + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Filter Operator + predicate: (_col8 = '2008-04-08') (type: boolean) + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Execution mode: llap - Map 4 + Map 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4595,31 +4614,6 @@ STAGE PLANS: value expressions: date (type: string) Execution mode: llap Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col8 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col8 = '2008-04-08') (type: boolean) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: uber Reduce Operator Tree: Group By Operator @@ -4812,9 +4806,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 3 <- Map 2 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4851,13 +4844,27 @@ STAGE PLANS: 0 Map 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Map-reduce partition columns: '13' (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 1 Map 4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Execution mode: llap - Map 3 + Map 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4866,27 +4873,13 @@ STAGE PLANS: Filter Operator predicate: (hr = 13) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 '13' (type: string) - 1 '13' (type: string) - input vertices: - 0 Map 2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + key expressions: '13' (type: string) + sort order: + + Map-reduce partition columns: '13' (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 4 + Reducer 3 Execution mode: uber Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 57fcc3c..792b674 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -1745,19 +1745,17 @@ Plan optimized by CBO. Vertex dependency in root stage Map 1 <- Union 2 (CONTAINS) Map 12 <- Union 10 (CONTAINS) -Map 14 <- Reducer 11 (BROADCAST_EDGE), Union 7 (CONTAINS) -Map 4 <- Union 2 (CONTAINS) -Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) +Map 6 <- Union 2 (CONTAINS) Map 9 <- Union 10 (CONTAINS) -Reducer 11 <- Map 13 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 5 <- Union 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 + Reducer 5 File Output Operator [FS_59] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -1766,8 +1764,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Map 14 [CONTAINS] + |<-Union 4 [SIMPLE_EDGE] + |<-Reducer 11 [CONTAINS] | Reduce Output Operator [RS_56] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -1785,90 +1783,89 @@ Stage-0 | | keys:{"Reducer 11":"_col2 (type: string)","Map 14":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 11 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_47] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 14 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_49] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col1 (type: string) - | | Map Join Operator [MAPJOIN_84] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} - | | | outputColumnNames:["_col1","_col2"] - | | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 13 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_44] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_38] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_80] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_37] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_36] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_35] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 10 [SIMPLE_EDGE] - | | |<-Map 12 [CONTAINS] - | | | Reduce Output Operator [RS_34] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_33] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_29] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_79] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_28] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 9 [CONTAINS] - | | Reduce Output Operator [RS_34] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_33] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_27] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_78] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_26] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_40] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_81] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_39] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 6 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_40] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_81] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_39] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_84] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} + | | outputColumnNames:["_col1","_col2"] + | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | |<-Map 13 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_44] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_38] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_80] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_37] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_36] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_35] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 10 [SIMPLE_EDGE] + | |<-Map 12 [CONTAINS] + | | Reduce Output Operator [RS_34] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_33] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_29] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_79] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_28] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map 9 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_27] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_78] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_26] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 3 [CONTAINS] Reduce Output Operator [RS_56] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -1883,92 +1880,91 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_83] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | keys:{"Reducer 3":"_col2 (type: string)","Map 8":"_col0 (type: string)"} | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [BROADCAST_EDGE] - | Reduce Output Operator [RS_21] - | key expressions:_col2 (type: string) - | Map-reduce partition columns:_col2 (type: string) + |<-Map 8 [BROADCAST_EDGE] + | Reduce Output Operator [RS_23] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) | sort order:+ - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col1 (type: string) - | Map Join Operator [MAPJOIN_82] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} - | | outputColumnNames:["_col1","_col2"] - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 5 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_18] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_12] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_76] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_11] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_10] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 2 [SIMPLE_EDGE] - | |<-Map 1 [CONTAINS] - | | Reduce Output Operator [RS_8] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_7] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_74] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 4 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_3] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_75] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_14] - outputColumnNames:["_col0"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_77] - predicate:key is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_13] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_14] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_77] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_13] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Map Join Operator [MAPJOIN_82] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"Reducer 3":"_col1 (type: string)","Map 7":"_col1 (type: string)"} + | outputColumnNames:["_col1","_col2"] + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + |<-Map 7 [BROADCAST_EDGE] + | Reduce Output Operator [RS_18] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_12] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_76] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_11] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_74] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_75] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, y.value @@ -2003,28 +1999,25 @@ Map 1 <- Union 2 (CONTAINS) Map 11 <- Union 12 (CONTAINS) Map 16 <- Union 12 (CONTAINS) Map 17 <- Union 14 (CONTAINS) -Map 19 <- Reducer 15 (BROADCAST_EDGE), Union 7 (CONTAINS) Map 20 <- Union 21 (CONTAINS) Map 27 <- Union 21 (CONTAINS) Map 28 <- Union 23 (CONTAINS) Map 29 <- Union 25 (CONTAINS) -Map 31 <- Reducer 26 (BROADCAST_EDGE), Union 9 (CONTAINS) -Map 4 <- Union 2 (CONTAINS) -Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) -Reducer 10 <- Union 9 (SIMPLE_EDGE) +Map 8 <- Union 2 (CONTAINS) Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) -Reducer 15 <- Map 18 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE) +Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) -Reducer 26 <- Map 30 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 8 <- Union 7 (SIMPLE_EDGE), Union 9 (CONTAINS) +Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 7 <- Union 6 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 10 + Reducer 7 File Output Operator [FS_119] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -2033,8 +2026,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 9 [SIMPLE_EDGE] - |<-Map 31 [CONTAINS] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 26 [CONTAINS] | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2052,157 +2045,158 @@ Stage-0 | | keys:{"Reducer 26":"_col2 (type: string)","Map 31":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 26 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_107] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 31 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_109] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_166] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 26":"_col1 (type: string)","Map 30":"_col1 (type: string)"} - | | | outputColumnNames:["_col2"] - | | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 30 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_104] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_98] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_160] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_97] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_96] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_95] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 25 [SIMPLE_EDGE] - | | |<-Map 29 [CONTAINS] - | | | Reduce Output Operator [RS_94] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_93] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_89] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_159] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_88] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 24 [CONTAINS] - | | Reduce Output Operator [RS_94] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_93] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_86] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 23 [SIMPLE_EDGE] - | | |<-Map 28 [CONTAINS] - | | | Reduce Output Operator [RS_85] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_84] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_80] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_158] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_79] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 22 [CONTAINS] - | | Reduce Output Operator [RS_85] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_84] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_77] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 21 [SIMPLE_EDGE] - | | |<-Map 20 [CONTAINS] - | | | Reduce Output Operator [RS_76] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_75] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_69] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_156] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_68] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 27 [CONTAINS] - | | Reduce Output Operator [RS_76] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_75] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_71] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_157] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_70] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_100] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_161] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_99] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 8 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_100] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_161] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_99] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_166] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 26":"_col1 (type: string)","Map 30":"_col1 (type: string)"} + | | outputColumnNames:["_col2"] + | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE + | |<-Map 30 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_104] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_98] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_160] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_97] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_96] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_95] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | |<-Union 25 [SIMPLE_EDGE] + | |<-Map 29 [CONTAINS] + | | Reduce Output Operator [RS_94] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_93] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_89] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_159] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_88] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 24 [CONTAINS] + | Reduce Output Operator [RS_94] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_93] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_86] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 23 [SIMPLE_EDGE] + | |<-Map 28 [CONTAINS] + | | Reduce Output Operator [RS_85] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_84] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_80] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_158] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_79] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 22 [CONTAINS] + | Reduce Output Operator [RS_85] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_84] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_77] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 21 [SIMPLE_EDGE] + | |<-Map 20 [CONTAINS] + | | Reduce Output Operator [RS_76] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_75] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_69] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_156] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_68] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 27 [CONTAINS] + | Reduce Output Operator [RS_76] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_75] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_71] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_157] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_70] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 5 [CONTAINS] Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2216,8 +2210,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Map 19 [CONTAINS] + |<-Union 4 [SIMPLE_EDGE] + |<-Reducer 15 [CONTAINS] | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2235,123 +2229,124 @@ Stage-0 | | keys:{"Reducer 15":"_col2 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 15 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_56] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 19 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_58] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_164] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} - | | | outputColumnNames:["_col2"] - | | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 18 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_53] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_47] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_154] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_46] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_45] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_44] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 14 [SIMPLE_EDGE] - | | |<-Map 17 [CONTAINS] - | | | Reduce Output Operator [RS_43] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_42] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_38] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_153] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_37] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 13 [CONTAINS] - | | Reduce Output Operator [RS_43] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_42] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_35] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 12 [SIMPLE_EDGE] - | | |<-Map 11 [CONTAINS] - | | | Reduce Output Operator [RS_34] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_33] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_27] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_151] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_26] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 16 [CONTAINS] - | | Reduce Output Operator [RS_34] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_33] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_29] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_152] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_28] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_49] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_155] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_48] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 6 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_49] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_155] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_48] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_164] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} + | | outputColumnNames:["_col2"] + | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + | |<-Map 18 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_53] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_47] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_154] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_46] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_45] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_44] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 14 [SIMPLE_EDGE] + | |<-Map 17 [CONTAINS] + | | Reduce Output Operator [RS_43] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_42] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_38] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_153] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_37] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 13 [CONTAINS] + | Reduce Output Operator [RS_43] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_42] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_35] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 12 [SIMPLE_EDGE] + | |<-Map 11 [CONTAINS] + | | Reduce Output Operator [RS_34] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_33] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_27] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_151] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_26] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 16 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_29] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_152] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_28] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 3 [CONTAINS] Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2366,91 +2361,92 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_163] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | keys:{"Reducer 3":"_col2 (type: string)","Map 10":"_col0 (type: string)"} | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [BROADCAST_EDGE] - | Reduce Output Operator [RS_21] - | key expressions:_col2 (type: string) - | Map-reduce partition columns:_col2 (type: string) + |<-Map 10 [BROADCAST_EDGE] + | Reduce Output Operator [RS_23] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) | sort order:+ - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_162] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} - | | outputColumnNames:["_col2"] - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 5 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_18] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_12] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_149] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_11] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_10] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 2 [SIMPLE_EDGE] - | |<-Map 1 [CONTAINS] - | | Reduce Output Operator [RS_8] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_7] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_147] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 4 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_3] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_148] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_14] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_150] - predicate:key is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_13] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string) + | Select Operator [SEL_14] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_150] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_13] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Map Join Operator [MAPJOIN_162] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"Reducer 3":"_col1 (type: string)","Map 9":"_col1 (type: string)"} + | outputColumnNames:["_col2"] + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + |<-Map 9 [BROADCAST_EDGE] + | Reduce Output Operator [RS_18] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_12] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_149] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_11] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_147] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 8 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_148] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -3033,26 +3029,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 12 <- Union 9 (CONTAINS) -Map 13 <- Union 9 (CONTAINS) -Map 17 <- Map 16 (BROADCAST_EDGE) -Map 18 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 19 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 20 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 21 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 8 <- Union 9 (CONTAINS) -Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Map 1 <- Union 2 (CONTAINS) +Map 13 <- Union 10 (CONTAINS) +Map 14 <- Union 10 (CONTAINS) +Map 18 <- Map 17 (BROADCAST_EDGE) +Map 19 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 20 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 21 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 22 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 6 <- Union 2 (CONTAINS) +Map 9 <- Union 10 (CONTAINS) +Reducer 11 <- Map 15 (SIMPLE_EDGE), Union 10 (SIMPLE_EDGE) +Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) +Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Union 4 - |<-Map 18 [CONTAINS] + Union 5 + |<-Map 19 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3062,10 +3059,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 18":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<-Map 17 [BROADCAST_EDGE] + | |<-Map 18 [BROADCAST_EDGE] | | Reduce Output Operator [RS_69] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3074,10 +3071,10 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_119] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 16":"_col0 (type: string)","Map 17":"_col0 (type: string)"} + | | | keys:{"Map 17":"_col0 (type: string)","Map 18":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 16 [BROADCAST_EDGE] + | | |<-Map 17 [BROADCAST_EDGE] | | | Reduce Output Operator [RS_64] | | | key expressions:_col0 (type: string) | | | Map-reduce partition columns:_col0 (type: string) @@ -3102,21 +3099,21 @@ Stage-0 | | TableScan [TS_49] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_125] + | | Reduce Output Operator [RS_123] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) | | Please refer to the previous Map Join Operator [MAPJOIN_119] - | | Reduce Output Operator [RS_126] + | | Reduce Output Operator [RS_124] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) | | Please refer to the previous Map Join Operator [MAPJOIN_119] - | | Reduce Output Operator [RS_127] + | | Reduce Output Operator [RS_125] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ @@ -3132,7 +3129,7 @@ Stage-0 | TableScan [TS_51] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 19 [CONTAINS] + |<-Map 20 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3142,10 +3139,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 19":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 20":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_54] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3155,7 +3152,7 @@ Stage-0 | TableScan [TS_53] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 20 [CONTAINS] + |<-Map 21 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3165,10 +3162,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 20":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 21":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_58] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3178,7 +3175,7 @@ Stage-0 | TableScan [TS_57] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 21 [CONTAINS] + |<-Map 22 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3188,10 +3185,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 21":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 22":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_61] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3201,7 +3198,7 @@ Stage-0 | TableScan [TS_60] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 11 [CONTAINS] + |<-Reducer 12 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3214,7 +3211,7 @@ Stage-0 | | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col4"] | | Statistics:Num rows: 620 Data size: 6547 Basic stats: COMPLETE Column stats: NONE - | |<-Map 15 [SIMPLE_EDGE] + | |<-Map 16 [SIMPLE_EDGE] | | Reduce Output Operator [RS_42] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3230,7 +3227,7 @@ Stage-0 | | TableScan [TS_32] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 10 [SIMPLE_EDGE] + | |<-Reducer 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_40] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3241,7 +3238,7 @@ Stage-0 | | keys:{"0":"_col0 (type: string)","1":"_col1 (type: string)"} | | outputColumnNames:["_col1"] | | Statistics:Num rows: 564 Data size: 5952 Basic stats: COMPLETE Column stats: NONE - | |<-Map 14 [SIMPLE_EDGE] + | |<-Map 15 [SIMPLE_EDGE] | | Reduce Output Operator [RS_37] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3257,8 +3254,8 @@ Stage-0 | | TableScan [TS_30] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Union 9 [SIMPLE_EDGE] - | |<-Map 12 [CONTAINS] + | |<-Union 10 [SIMPLE_EDGE] + | |<-Map 13 [CONTAINS] | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3273,7 +3270,7 @@ Stage-0 | | TableScan [TS_23] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 13 [CONTAINS] + | |<-Map 14 [CONTAINS] | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3288,7 +3285,7 @@ Stage-0 | | TableScan [TS_27] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 8 [CONTAINS] + | |<-Map 9 [CONTAINS] | Reduce Output Operator [RS_35] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3303,7 +3300,7 @@ Stage-0 | TableScan [TS_21] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [CONTAINS] + |<-Reducer 4 [CONTAINS] File Output Operator [FS_75] compressed:false Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3316,7 +3313,7 @@ Stage-0 | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col1","_col4"] | Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - |<-Map 7 [SIMPLE_EDGE] + |<-Map 8 [SIMPLE_EDGE] | Reduce Output Operator [RS_18] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3332,71 +3329,64 @@ Stage-0 | TableScan [TS_8] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] - | Reduce Output Operator [RS_16] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_115] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 1":"_col0 (type: string)","Map 6":"_col1 (type: string)"} - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | |<-Map 6 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_13] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_7] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_102] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_6] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_121] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Please refer to the previous Select Operator [SEL_7] - | |<-Select Operator [SEL_1] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_100] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 5 [CONTAINS] - Reduce Output Operator [RS_16] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_115] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 5":"_col0 (type: string)","Map 6":"_col1 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - |<- Please refer to the previous Map 6 [BROADCAST_EDGE] - |<-Select Operator [SEL_3] - outputColumnNames:["_col0"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_101] - predicate:value is not null (type: boolean) + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) + sort order:+ + Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_115] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"0":"_col0 (type: string)","1":"_col1 (type: string)"} + | outputColumnNames:["_col1"] + | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + |<-Map 7 [SIMPLE_EDGE] + | Reduce Output Operator [RS_13] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_7] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_102] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_6] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_11] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) + | sort order:+ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_100] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_11] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_101] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, y.value @@ -3428,32 +3418,33 @@ Plan optimized by CBO. Vertex dependency in root stage Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 24 <- Map 23 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 24 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Map 10 <- Union 2 (CONTAINS) +Map 13 <- Union 14 (CONTAINS) +Map 20 <- Union 14 (CONTAINS) +Map 21 <- Union 16 (CONTAINS) +Map 25 <- Map 24 (BROADCAST_EDGE) +Map 26 <- Union 27 (CONTAINS) +Map 33 <- Union 27 (CONTAINS) +Map 34 <- Union 29 (CONTAINS) +Map 35 <- Union 31 (CONTAINS) +Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) +Reducer 17 <- Union 16 (SIMPLE_EDGE) +Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) +Reducer 3 <- Union 2 (SIMPLE_EDGE) +Reducer 30 <- Union 29 (SIMPLE_EDGE), Union 31 (CONTAINS) +Reducer 32 <- Map 25 (BROADCAST_EDGE), Union 31 (SIMPLE_EDGE), Union 8 (CONTAINS) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) +Reducer 9 <- Union 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 + Reducer 9 File Output Operator [FS_119] compressed:false Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE @@ -3462,8 +3453,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Reducer 31 [CONTAINS] + |<-Union 8 [SIMPLE_EDGE] + |<-Reducer 32 [CONTAINS] | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3478,10 +3469,10 @@ Stage-0 | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_164] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 24":"_col1 (type: string)","Reducer 31":"_col1 (type: string)"} + | | keys:{"Map 25":"_col1 (type: string)","Reducer 32":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | |<-Map 24 [BROADCAST_EDGE] + | |<-Map 25 [BROADCAST_EDGE] | | Reduce Output Operator [RS_107] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3490,10 +3481,10 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_163] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 23":"_col0 (type: string)","Map 24":"_col0 (type: string)"} + | | | keys:{"Map 24":"_col0 (type: string)","Map 25":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 23 [BROADCAST_EDGE] + | | |<-Map 24 [BROADCAST_EDGE] | | | Reduce Output Operator [RS_102] | | | key expressions:_col0 (type: string) | | | Map-reduce partition columns:_col0 (type: string) @@ -3525,8 +3516,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | |<-Union 30 [SIMPLE_EDGE] - | |<-Map 34 [CONTAINS] + | |<-Union 31 [SIMPLE_EDGE] + | |<-Map 35 [CONTAINS] | | Reduce Output Operator [RS_98] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3545,7 +3536,7 @@ Stage-0 | | TableScan [TS_92] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 29 [CONTAINS] + | |<-Reducer 30 [CONTAINS] | Reduce Output Operator [RS_98] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3559,8 +3550,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | |<-Union 28 [SIMPLE_EDGE] - | |<-Map 33 [CONTAINS] + | |<-Union 29 [SIMPLE_EDGE] + | |<-Map 34 [CONTAINS] | | Reduce Output Operator [RS_89] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3579,7 +3570,7 @@ Stage-0 | | TableScan [TS_83] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 27 [CONTAINS] + | |<-Reducer 28 [CONTAINS] | Reduce Output Operator [RS_89] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3593,8 +3584,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 26 [SIMPLE_EDGE] - | |<-Map 25 [CONTAINS] + | |<-Union 27 [SIMPLE_EDGE] + | |<-Map 26 [CONTAINS] | | Reduce Output Operator [RS_80] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3613,7 +3604,7 @@ Stage-0 | | TableScan [TS_72] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 32 [CONTAINS] + | |<-Map 33 [CONTAINS] | Reduce Output Operator [RS_80] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3632,7 +3623,7 @@ Stage-0 | TableScan [TS_74] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 6 [CONTAINS] + |<-Reducer 7 [CONTAINS] Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3646,8 +3637,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 5 [SIMPLE_EDGE] - |<-Reducer 18 [CONTAINS] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 19 [CONTAINS] | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3665,7 +3656,7 @@ Stage-0 | | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 22 [SIMPLE_EDGE] + | |<-Map 23 [SIMPLE_EDGE] | | Reduce Output Operator [RS_58] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3681,7 +3672,7 @@ Stage-0 | | TableScan [TS_48] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 17 [SIMPLE_EDGE] + | |<-Reducer 18 [SIMPLE_EDGE] | Reduce Output Operator [RS_56] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) @@ -3692,7 +3683,7 @@ Stage-0 | | keys:{"0":"_col1 (type: string)","1":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | |<-Map 21 [SIMPLE_EDGE] + | |<-Map 22 [SIMPLE_EDGE] | | Reduce Output Operator [RS_53] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3708,7 +3699,7 @@ Stage-0 | | TableScan [TS_46] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 16 [SIMPLE_EDGE] + | |<-Reducer 17 [SIMPLE_EDGE] | Reduce Output Operator [RS_51] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3721,8 +3712,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | |<-Union 15 [SIMPLE_EDGE] - | |<-Map 20 [CONTAINS] + | |<-Union 16 [SIMPLE_EDGE] + | |<-Map 21 [CONTAINS] | | Reduce Output Operator [RS_43] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3741,7 +3732,7 @@ Stage-0 | | TableScan [TS_37] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 14 [CONTAINS] + | |<-Reducer 15 [CONTAINS] | Reduce Output Operator [RS_43] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3755,8 +3746,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 13 [SIMPLE_EDGE] - | |<-Map 12 [CONTAINS] + | |<-Union 14 [SIMPLE_EDGE] + | |<-Map 13 [CONTAINS] | | Reduce Output Operator [RS_34] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3775,7 +3766,7 @@ Stage-0 | | TableScan [TS_26] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 19 [CONTAINS] + | |<-Map 20 [CONTAINS] | Reduce Output Operator [RS_34] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3794,7 +3785,7 @@ Stage-0 | TableScan [TS_28] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 4 [CONTAINS] + |<-Reducer 5 [CONTAINS] Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3812,7 +3803,7 @@ Stage-0 | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 11 [SIMPLE_EDGE] + |<-Map 12 [SIMPLE_EDGE] | Reduce Output Operator [RS_23] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3828,18 +3819,18 @@ Stage-0 | TableScan [TS_13] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [SIMPLE_EDGE] + |<-Reducer 4 [SIMPLE_EDGE] Reduce Output Operator [RS_21] key expressions:_col2 (type: string) Map-reduce partition columns:_col2 (type: string) sort order:+ Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_159] + Merge Join Operator [MERGEJOIN_159] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col1 (type: string)","Map 10":"_col1 (type: string)"} + | keys:{"0":"_col1 (type: string)","1":"_col1 (type: string)"} | outputColumnNames:["_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - |<-Map 10 [BROADCAST_EDGE] + |<-Map 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_18] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3855,52 +3846,58 @@ Stage-0 | TableScan [TS_11] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_10] - outputColumnNames:["_col1"] + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) + sort order:+ Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_9] - | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_1] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_144] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 9 [CONTAINS] - Reduce Output Operator [RS_8] - key expressions:_col0 (type: string), _col1 (type: string) - Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - sort order:++ - Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_7] - keys:_col0 (type: string), _col1 (type: string) - outputColumnNames:["_col0","_col1"] + Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_144] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 10 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_3] + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_145] - predicate:value is not null (type: boolean) + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_145] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index d0200d2..7026cd4 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -1688,9 +1688,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1714,7 +1713,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Map 3 + Map 4 Map Operator Tree: TableScan alias: src @@ -1726,22 +1725,12 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: llap Reducer 2 Execution mode: llap @@ -1752,13 +1741,23 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 4 + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 1 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Reducer 3 Execution mode: uber Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 3e82696..e69c2eb 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -4262,7 +4262,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4277,9 +4277,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4290,26 +4289,11 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart @@ -4330,7 +4314,35 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 2 + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: vectorized, uber Reduce Operator Tree: Group By Operator @@ -4345,19 +4357,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -4365,7 +4364,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4567,8 +4566,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4576,13 +4575,33 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 ds (type: string) + 1 ds (type: string) + outputColumnNames: _col8 + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Filter Operator + predicate: (_col8 = '2008-04-08') (type: boolean) + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Execution mode: llap - Map 4 + Map 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4595,31 +4614,6 @@ STAGE PLANS: value expressions: date (type: string) Execution mode: vectorized, llap Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col8 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col8 = '2008-04-08') (type: boolean) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: vectorized, uber Reduce Operator Tree: Group By Operator @@ -4812,9 +4806,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 3 <- Map 2 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4851,13 +4844,27 @@ STAGE PLANS: 0 Map 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Map-reduce partition columns: '13' (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 1 Map 4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Execution mode: vectorized, llap - Map 3 + Map 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4866,27 +4873,13 @@ STAGE PLANS: Filter Operator predicate: (hr = 13) (type: boolean) Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 '13' (type: string) - 1 '13' (type: string) - input vertices: - 0 Map 2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + key expressions: '13' (type: string) + sort order: + + Map-reduce partition columns: '13' (type: string) + Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 4 + Reducer 3 Execution mode: vectorized, uber Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out index 7194be5..0d22ea7 100644 --- ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out +++ ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out @@ -245,9 +245,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -271,7 +270,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 3 + Map 4 Map Operator Tree: TableScan alias: a @@ -283,25 +282,11 @@ STAGE PLANS: expressions: key (type: int) outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - input vertices: - 0 Reducer 2 - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -314,12 +299,26 @@ STAGE PLANS: expressions: _col0 (type: int) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + input vertices: + 1 Map 4 + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out index b83aa27..e27a006 100644 --- ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out @@ -138,7 +138,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -211,9 +211,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 3 <- Map 2 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -309,13 +308,40 @@ STAGE PLANS: Position of Big Table: 1 Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - key expressions: _col6 (type: string), _col6 (type: string) - sort order: ++ - Map-reduce partition columns: _col6 (type: string), _col6 (type: string) - Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Map 4 => 58 + keys: + 0 _col6 (type: string), _col6 (type: string) + 1 key (type: string), key (type: string) + input vertices: + 1 Map 4 + Position of Big Table: 0 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Map 5 => 1 + keys: + 0 + 1 + input vertices: + 1 Map 5 + Position of Big Table: 0 + Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -369,7 +395,7 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [b] - Map 3 + Map 4 Map Operator Tree: TableScan alias: c @@ -379,40 +405,13 @@ STAGE PLANS: isSamplingPred: false predicate: key is not null (type: boolean) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Estimated key counts: Map 2 => 1 - keys: - 0 _col6 (type: string), _col6 (type: string) - 1 key (type: string), key (type: string) - input vertices: - 0 Map 2 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Map Join Operator - condition map: - Inner Join 0 to 1 - Estimated key counts: Map 5 => 1 - keys: - 0 - 1 - input vertices: - 1 Map 5 - Position of Big Table: 0 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false + Reduce Output Operator + key expressions: key (type: string), key (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), key (type: string) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -579,7 +578,7 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [d] - Reducer 4 + Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -615,7 +614,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index af5e6e6..3ccc52f 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -323,8 +323,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -355,28 +354,12 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -389,12 +372,28 @@ STAGE PLANS: expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -543,8 +542,7 @@ STAGE PLANS: Tez Edges: Map 1 <- Map 3 (CUSTOM_EDGE) - Map 4 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -599,28 +597,12 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -633,12 +615,28 @@ STAGE PLANS: expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -848,8 +846,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -880,28 +877,12 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -910,12 +891,28 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -941,8 +938,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -967,28 +963,12 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -997,12 +977,28 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out index 3f980b6..c92f030 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out @@ -447,7 +447,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Map 1 <- Map 2 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -462,23 +462,6 @@ STAGE PLANS: expressions: key (type: int) outputColumnNames: _col0 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Map 2 - Map Operator Tree: - TableScan - alias: tab_part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 2) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Right Outer Join0 to 1 @@ -487,7 +470,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 1 Map 2 Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true File Output Operator @@ -497,6 +480,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 2 + Map Operator Tree: + TableScan + alias: tab_part + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -516,8 +516,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -546,24 +545,11 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -571,11 +557,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -595,8 +594,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -625,28 +623,12 @@ STAGE PLANS: Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col2 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: _col0 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -654,12 +636,28 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col2 + input vertices: + 1 Map 3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out index f30caa1..5e8015d 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out @@ -90,7 +90,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[18][bigTable=a] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -103,8 +103,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -126,10 +125,27 @@ STAGE PLANS: 1 Map 2 Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + input vertices: + 1 Map 3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan @@ -149,27 +165,10 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - input vertices: - 0 Map 1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) Stage: Stage-0 Fetch Operator @@ -177,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=a] in task 'Map 4' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -197,8 +196,7 @@ STAGE PLANS: Tez Edges: Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -248,27 +246,10 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col5 - input vertices: - 1 Reducer 2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -276,10 +257,27 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col5 + input vertices: + 0 Map 4 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -288,7 +286,7 @@ STAGE PLANS: ListSink Warning: Map Join MAPJOIN[17][bigTable=d1] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[18][bigTable=a] in task 'Map 4' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -302,8 +300,7 @@ STAGE PLANS: Tez Edges: Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -344,27 +341,10 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col5 - input vertices: - 1 Reducer 2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -372,10 +352,27 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col5 + input vertices: + 0 Map 4 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -383,7 +380,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Reducer 4' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -401,8 +398,8 @@ STAGE PLANS: Tez Edges: Map 3 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -473,6 +470,17 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -481,7 +489,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1 input vertices: - 1 Reducer 4 + 0 Reducer 2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -490,17 +498,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index b6fa1ac..93bf452 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -4125,7 +4125,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4140,9 +4140,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4153,25 +4152,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map 2 Map Operator Tree: TableScan alias: srcpart @@ -4191,7 +4175,34 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -4205,18 +4216,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -4224,7 +4223,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4420,8 +4419,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4429,12 +4428,32 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map 4 + Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 ds (type: string) + 1 ds (type: string) + outputColumnNames: _col8 + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Filter Operator + predicate: (_col8 = '2008-04-08') (type: boolean) + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Map 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4447,30 +4466,6 @@ STAGE PLANS: value expressions: date (type: string) Reducer 2 Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col8 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col8 = '2008-04-08') (type: boolean) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial @@ -4657,9 +4652,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 3 <- Map 2 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4695,12 +4689,26 @@ STAGE PLANS: 0 Map 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Map-reduce partition columns: '13' (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 1 Map 4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Map 3 + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Map 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4709,26 +4717,12 @@ STAGE PLANS: Filter Operator predicate: (hr = 13) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 '13' (type: string) - 1 '13' (type: string) - input vertices: - 0 Map 2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 + Reduce Output Operator + key expressions: '13' (type: string) + sort order: + + Map-reduce partition columns: '13' (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/tez/explainuser_2.q.out ql/src/test/results/clientpositive/tez/explainuser_2.q.out index 57fcc3c..792b674 100644 --- ql/src/test/results/clientpositive/tez/explainuser_2.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_2.q.out @@ -1745,19 +1745,17 @@ Plan optimized by CBO. Vertex dependency in root stage Map 1 <- Union 2 (CONTAINS) Map 12 <- Union 10 (CONTAINS) -Map 14 <- Reducer 11 (BROADCAST_EDGE), Union 7 (CONTAINS) -Map 4 <- Union 2 (CONTAINS) -Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) +Map 6 <- Union 2 (CONTAINS) Map 9 <- Union 10 (CONTAINS) -Reducer 11 <- Map 13 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 5 <- Union 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 + Reducer 5 File Output Operator [FS_59] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -1766,8 +1764,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Map 14 [CONTAINS] + |<-Union 4 [SIMPLE_EDGE] + |<-Reducer 11 [CONTAINS] | Reduce Output Operator [RS_56] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -1785,90 +1783,89 @@ Stage-0 | | keys:{"Reducer 11":"_col2 (type: string)","Map 14":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 11 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_47] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 14 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_49] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col1 (type: string) - | | Map Join Operator [MAPJOIN_84] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} - | | | outputColumnNames:["_col1","_col2"] - | | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 13 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_44] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_38] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_80] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_37] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_36] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_35] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 10 [SIMPLE_EDGE] - | | |<-Map 12 [CONTAINS] - | | | Reduce Output Operator [RS_34] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_33] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_29] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_79] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_28] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 9 [CONTAINS] - | | Reduce Output Operator [RS_34] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_33] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_27] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_78] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_26] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_40] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_81] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_39] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 6 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_40] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_81] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_39] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_84] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} + | | outputColumnNames:["_col1","_col2"] + | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | |<-Map 13 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_44] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_38] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_80] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_37] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_36] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_35] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 10 [SIMPLE_EDGE] + | |<-Map 12 [CONTAINS] + | | Reduce Output Operator [RS_34] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_33] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_29] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_79] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_28] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map 9 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_27] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_78] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_26] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 3 [CONTAINS] Reduce Output Operator [RS_56] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -1883,92 +1880,91 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_83] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | keys:{"Reducer 3":"_col2 (type: string)","Map 8":"_col0 (type: string)"} | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [BROADCAST_EDGE] - | Reduce Output Operator [RS_21] - | key expressions:_col2 (type: string) - | Map-reduce partition columns:_col2 (type: string) + |<-Map 8 [BROADCAST_EDGE] + | Reduce Output Operator [RS_23] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) | sort order:+ - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col1 (type: string) - | Map Join Operator [MAPJOIN_82] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} - | | outputColumnNames:["_col1","_col2"] - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 5 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_18] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_12] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_76] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_11] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_10] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 2 [SIMPLE_EDGE] - | |<-Map 1 [CONTAINS] - | | Reduce Output Operator [RS_8] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_7] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_74] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 4 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_3] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_75] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_14] - outputColumnNames:["_col0"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_77] - predicate:key is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_13] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_14] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_77] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_13] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Map Join Operator [MAPJOIN_82] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"Reducer 3":"_col1 (type: string)","Map 7":"_col1 (type: string)"} + | outputColumnNames:["_col1","_col2"] + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + |<-Map 7 [BROADCAST_EDGE] + | Reduce Output Operator [RS_18] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_12] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_76] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_11] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_74] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_75] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, y.value @@ -2003,28 +1999,25 @@ Map 1 <- Union 2 (CONTAINS) Map 11 <- Union 12 (CONTAINS) Map 16 <- Union 12 (CONTAINS) Map 17 <- Union 14 (CONTAINS) -Map 19 <- Reducer 15 (BROADCAST_EDGE), Union 7 (CONTAINS) Map 20 <- Union 21 (CONTAINS) Map 27 <- Union 21 (CONTAINS) Map 28 <- Union 23 (CONTAINS) Map 29 <- Union 25 (CONTAINS) -Map 31 <- Reducer 26 (BROADCAST_EDGE), Union 9 (CONTAINS) -Map 4 <- Union 2 (CONTAINS) -Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) -Reducer 10 <- Union 9 (SIMPLE_EDGE) +Map 8 <- Union 2 (CONTAINS) Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) -Reducer 15 <- Map 18 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE) +Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS) Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) -Reducer 26 <- Map 30 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 8 <- Union 7 (SIMPLE_EDGE), Union 9 (CONTAINS) +Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 7 <- Union 6 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 10 + Reducer 7 File Output Operator [FS_119] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -2033,8 +2026,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 9 [SIMPLE_EDGE] - |<-Map 31 [CONTAINS] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 26 [CONTAINS] | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2052,157 +2045,158 @@ Stage-0 | | keys:{"Reducer 26":"_col2 (type: string)","Map 31":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 26 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_107] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 31 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_109] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_166] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 26":"_col1 (type: string)","Map 30":"_col1 (type: string)"} - | | | outputColumnNames:["_col2"] - | | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 30 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_104] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_98] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_160] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_97] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_96] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_95] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 25 [SIMPLE_EDGE] - | | |<-Map 29 [CONTAINS] - | | | Reduce Output Operator [RS_94] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_93] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_89] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_159] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_88] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 24 [CONTAINS] - | | Reduce Output Operator [RS_94] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_93] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_86] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 23 [SIMPLE_EDGE] - | | |<-Map 28 [CONTAINS] - | | | Reduce Output Operator [RS_85] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_84] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_80] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_158] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_79] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 22 [CONTAINS] - | | Reduce Output Operator [RS_85] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_84] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_77] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 21 [SIMPLE_EDGE] - | | |<-Map 20 [CONTAINS] - | | | Reduce Output Operator [RS_76] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_75] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_69] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_156] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_68] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 27 [CONTAINS] - | | Reduce Output Operator [RS_76] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_75] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_71] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_157] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_70] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_100] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_161] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_99] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 8 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_100] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_161] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_99] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_166] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 26":"_col1 (type: string)","Map 30":"_col1 (type: string)"} + | | outputColumnNames:["_col2"] + | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE + | |<-Map 30 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_104] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_98] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_160] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_97] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_96] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_95] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | |<-Union 25 [SIMPLE_EDGE] + | |<-Map 29 [CONTAINS] + | | Reduce Output Operator [RS_94] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_93] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_89] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_159] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_88] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 24 [CONTAINS] + | Reduce Output Operator [RS_94] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_93] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_86] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 23 [SIMPLE_EDGE] + | |<-Map 28 [CONTAINS] + | | Reduce Output Operator [RS_85] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_84] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_80] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_158] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_79] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 22 [CONTAINS] + | Reduce Output Operator [RS_85] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_84] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_77] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 21 [SIMPLE_EDGE] + | |<-Map 20 [CONTAINS] + | | Reduce Output Operator [RS_76] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_75] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_69] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_156] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_68] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 27 [CONTAINS] + | Reduce Output Operator [RS_76] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_75] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_71] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_157] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_70] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 5 [CONTAINS] Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2216,8 +2210,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Map 19 [CONTAINS] + |<-Union 4 [SIMPLE_EDGE] + |<-Reducer 15 [CONTAINS] | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2235,123 +2229,124 @@ Stage-0 | | keys:{"Reducer 15":"_col2 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 15 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_56] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | |<-Map 19 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_58] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ - | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_164] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} - | | | outputColumnNames:["_col2"] - | | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 18 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_53] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_47] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_154] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_46] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_45] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_44] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 14 [SIMPLE_EDGE] - | | |<-Map 17 [CONTAINS] - | | | Reduce Output Operator [RS_43] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_42] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_38] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_153] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_37] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 13 [CONTAINS] - | | Reduce Output Operator [RS_43] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_42] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_35] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 12 [SIMPLE_EDGE] - | | |<-Map 11 [CONTAINS] - | | | Reduce Output Operator [RS_34] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_33] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_27] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_151] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_26] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 16 [CONTAINS] - | | Reduce Output Operator [RS_34] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_33] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_29] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_152] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_28] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_49] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_155] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_48] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 6 [CONTAINS] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_49] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_155] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_48] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Map Join Operator [MAPJOIN_164] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} + | | outputColumnNames:["_col2"] + | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + | |<-Map 18 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_53] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_47] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_154] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_46] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_45] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_44] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 14 [SIMPLE_EDGE] + | |<-Map 17 [CONTAINS] + | | Reduce Output Operator [RS_43] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_42] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_38] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_153] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_37] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 13 [CONTAINS] + | Reduce Output Operator [RS_43] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_42] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_35] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 12 [SIMPLE_EDGE] + | |<-Map 11 [CONTAINS] + | | Reduce Output Operator [RS_34] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_33] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_27] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_151] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_26] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 16 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_29] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_152] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_28] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 3 [CONTAINS] Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -2366,91 +2361,92 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_163] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | keys:{"Reducer 3":"_col2 (type: string)","Map 10":"_col0 (type: string)"} | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [BROADCAST_EDGE] - | Reduce Output Operator [RS_21] - | key expressions:_col2 (type: string) - | Map-reduce partition columns:_col2 (type: string) + |<-Map 10 [BROADCAST_EDGE] + | Reduce Output Operator [RS_23] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) | sort order:+ - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_162] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} - | | outputColumnNames:["_col2"] - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 5 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_18] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_12] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_149] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_11] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_10] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 2 [SIMPLE_EDGE] - | |<-Map 1 [CONTAINS] - | | Reduce Output Operator [RS_8] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_7] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_147] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 4 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_3] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_148] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_14] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_150] - predicate:key is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_13] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string) + | Select Operator [SEL_14] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_150] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_13] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Map Join Operator [MAPJOIN_162] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"Reducer 3":"_col1 (type: string)","Map 9":"_col1 (type: string)"} + | outputColumnNames:["_col2"] + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + |<-Map 9 [BROADCAST_EDGE] + | Reduce Output Operator [RS_18] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_12] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_149] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_11] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_147] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 8 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_148] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -3033,26 +3029,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 12 <- Union 9 (CONTAINS) -Map 13 <- Union 9 (CONTAINS) -Map 17 <- Map 16 (BROADCAST_EDGE) -Map 18 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 19 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 20 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 21 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 8 <- Union 9 (CONTAINS) -Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +Map 1 <- Union 2 (CONTAINS) +Map 13 <- Union 10 (CONTAINS) +Map 14 <- Union 10 (CONTAINS) +Map 18 <- Map 17 (BROADCAST_EDGE) +Map 19 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 20 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 21 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 22 <- Map 18 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 6 <- Union 2 (CONTAINS) +Map 9 <- Union 10 (CONTAINS) +Reducer 11 <- Map 15 (SIMPLE_EDGE), Union 10 (SIMPLE_EDGE) +Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) +Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Union 4 - |<-Map 18 [CONTAINS] + Union 5 + |<-Map 19 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3062,10 +3059,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 18":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<-Map 17 [BROADCAST_EDGE] + | |<-Map 18 [BROADCAST_EDGE] | | Reduce Output Operator [RS_69] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3074,10 +3071,10 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_119] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 16":"_col0 (type: string)","Map 17":"_col0 (type: string)"} + | | | keys:{"Map 17":"_col0 (type: string)","Map 18":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 16 [BROADCAST_EDGE] + | | |<-Map 17 [BROADCAST_EDGE] | | | Reduce Output Operator [RS_64] | | | key expressions:_col0 (type: string) | | | Map-reduce partition columns:_col0 (type: string) @@ -3102,21 +3099,21 @@ Stage-0 | | TableScan [TS_49] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_125] + | | Reduce Output Operator [RS_123] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) | | Please refer to the previous Map Join Operator [MAPJOIN_119] - | | Reduce Output Operator [RS_126] + | | Reduce Output Operator [RS_124] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) | | Please refer to the previous Map Join Operator [MAPJOIN_119] - | | Reduce Output Operator [RS_127] + | | Reduce Output Operator [RS_125] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ @@ -3132,7 +3129,7 @@ Stage-0 | TableScan [TS_51] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 19 [CONTAINS] + |<-Map 20 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3142,10 +3139,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 19":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 20":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_54] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3155,7 +3152,7 @@ Stage-0 | TableScan [TS_53] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 20 [CONTAINS] + |<-Map 21 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3165,10 +3162,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 20":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 21":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_58] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3178,7 +3175,7 @@ Stage-0 | TableScan [TS_57] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 21 [CONTAINS] + |<-Map 22 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3188,10 +3185,10 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 17":"_col1 (type: string)","Map 21":"_col0 (type: string)"} + | | keys:{"Map 18":"_col1 (type: string)","Map 22":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<- Please refer to the previous Map 18 [BROADCAST_EDGE] | |<-Select Operator [SEL_61] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -3201,7 +3198,7 @@ Stage-0 | TableScan [TS_60] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 11 [CONTAINS] + |<-Reducer 12 [CONTAINS] | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3214,7 +3211,7 @@ Stage-0 | | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col4"] | | Statistics:Num rows: 620 Data size: 6547 Basic stats: COMPLETE Column stats: NONE - | |<-Map 15 [SIMPLE_EDGE] + | |<-Map 16 [SIMPLE_EDGE] | | Reduce Output Operator [RS_42] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3230,7 +3227,7 @@ Stage-0 | | TableScan [TS_32] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 10 [SIMPLE_EDGE] + | |<-Reducer 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_40] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3241,7 +3238,7 @@ Stage-0 | | keys:{"0":"_col0 (type: string)","1":"_col1 (type: string)"} | | outputColumnNames:["_col1"] | | Statistics:Num rows: 564 Data size: 5952 Basic stats: COMPLETE Column stats: NONE - | |<-Map 14 [SIMPLE_EDGE] + | |<-Map 15 [SIMPLE_EDGE] | | Reduce Output Operator [RS_37] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3257,8 +3254,8 @@ Stage-0 | | TableScan [TS_30] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Union 9 [SIMPLE_EDGE] - | |<-Map 12 [CONTAINS] + | |<-Union 10 [SIMPLE_EDGE] + | |<-Map 13 [CONTAINS] | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3273,7 +3270,7 @@ Stage-0 | | TableScan [TS_23] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 13 [CONTAINS] + | |<-Map 14 [CONTAINS] | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3288,7 +3285,7 @@ Stage-0 | | TableScan [TS_27] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 8 [CONTAINS] + | |<-Map 9 [CONTAINS] | Reduce Output Operator [RS_35] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3303,7 +3300,7 @@ Stage-0 | TableScan [TS_21] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [CONTAINS] + |<-Reducer 4 [CONTAINS] File Output Operator [FS_75] compressed:false Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE @@ -3316,7 +3313,7 @@ Stage-0 | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col1","_col4"] | Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - |<-Map 7 [SIMPLE_EDGE] + |<-Map 8 [SIMPLE_EDGE] | Reduce Output Operator [RS_18] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3332,71 +3329,64 @@ Stage-0 | TableScan [TS_8] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] - | Reduce Output Operator [RS_16] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_115] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 1":"_col0 (type: string)","Map 6":"_col1 (type: string)"} - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | |<-Map 6 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_13] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_7] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_102] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_6] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_121] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Please refer to the previous Select Operator [SEL_7] - | |<-Select Operator [SEL_1] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_100] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 5 [CONTAINS] - Reduce Output Operator [RS_16] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_115] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 5":"_col0 (type: string)","Map 6":"_col1 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - |<- Please refer to the previous Map 6 [BROADCAST_EDGE] - |<-Select Operator [SEL_3] - outputColumnNames:["_col0"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_101] - predicate:value is not null (type: boolean) + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) + sort order:+ + Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_115] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"0":"_col0 (type: string)","1":"_col1 (type: string)"} + | outputColumnNames:["_col1"] + | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + |<-Map 7 [SIMPLE_EDGE] + | Reduce Output Operator [RS_13] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_7] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_102] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_6] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_11] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) + | sort order:+ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_100] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_11] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_101] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, y.value @@ -3428,32 +3418,33 @@ Plan optimized by CBO. Vertex dependency in root stage Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 24 <- Map 23 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 24 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Map 10 <- Union 2 (CONTAINS) +Map 13 <- Union 14 (CONTAINS) +Map 20 <- Union 14 (CONTAINS) +Map 21 <- Union 16 (CONTAINS) +Map 25 <- Map 24 (BROADCAST_EDGE) +Map 26 <- Union 27 (CONTAINS) +Map 33 <- Union 27 (CONTAINS) +Map 34 <- Union 29 (CONTAINS) +Map 35 <- Union 31 (CONTAINS) +Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) +Reducer 17 <- Union 16 (SIMPLE_EDGE) +Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) +Reducer 3 <- Union 2 (SIMPLE_EDGE) +Reducer 30 <- Union 29 (SIMPLE_EDGE), Union 31 (CONTAINS) +Reducer 32 <- Map 25 (BROADCAST_EDGE), Union 31 (SIMPLE_EDGE), Union 8 (CONTAINS) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) +Reducer 9 <- Union 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 + Reducer 9 File Output Operator [FS_119] compressed:false Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE @@ -3462,8 +3453,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE - |<-Union 7 [SIMPLE_EDGE] - |<-Reducer 31 [CONTAINS] + |<-Union 8 [SIMPLE_EDGE] + |<-Reducer 32 [CONTAINS] | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3478,10 +3469,10 @@ Stage-0 | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_164] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 24":"_col1 (type: string)","Reducer 31":"_col1 (type: string)"} + | | keys:{"Map 25":"_col1 (type: string)","Reducer 32":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | |<-Map 24 [BROADCAST_EDGE] + | |<-Map 25 [BROADCAST_EDGE] | | Reduce Output Operator [RS_107] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3490,10 +3481,10 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_163] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 23":"_col0 (type: string)","Map 24":"_col0 (type: string)"} + | | | keys:{"Map 24":"_col0 (type: string)","Map 25":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 23 [BROADCAST_EDGE] + | | |<-Map 24 [BROADCAST_EDGE] | | | Reduce Output Operator [RS_102] | | | key expressions:_col0 (type: string) | | | Map-reduce partition columns:_col0 (type: string) @@ -3525,8 +3516,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | |<-Union 30 [SIMPLE_EDGE] - | |<-Map 34 [CONTAINS] + | |<-Union 31 [SIMPLE_EDGE] + | |<-Map 35 [CONTAINS] | | Reduce Output Operator [RS_98] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3545,7 +3536,7 @@ Stage-0 | | TableScan [TS_92] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 29 [CONTAINS] + | |<-Reducer 30 [CONTAINS] | Reduce Output Operator [RS_98] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3559,8 +3550,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | |<-Union 28 [SIMPLE_EDGE] - | |<-Map 33 [CONTAINS] + | |<-Union 29 [SIMPLE_EDGE] + | |<-Map 34 [CONTAINS] | | Reduce Output Operator [RS_89] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3579,7 +3570,7 @@ Stage-0 | | TableScan [TS_83] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 27 [CONTAINS] + | |<-Reducer 28 [CONTAINS] | Reduce Output Operator [RS_89] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3593,8 +3584,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 26 [SIMPLE_EDGE] - | |<-Map 25 [CONTAINS] + | |<-Union 27 [SIMPLE_EDGE] + | |<-Map 26 [CONTAINS] | | Reduce Output Operator [RS_80] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3613,7 +3604,7 @@ Stage-0 | | TableScan [TS_72] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 32 [CONTAINS] + | |<-Map 33 [CONTAINS] | Reduce Output Operator [RS_80] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3632,7 +3623,7 @@ Stage-0 | TableScan [TS_74] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 6 [CONTAINS] + |<-Reducer 7 [CONTAINS] Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3646,8 +3637,8 @@ Stage-0 | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 5 [SIMPLE_EDGE] - |<-Reducer 18 [CONTAINS] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 19 [CONTAINS] | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3665,7 +3656,7 @@ Stage-0 | | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 22 [SIMPLE_EDGE] + | |<-Map 23 [SIMPLE_EDGE] | | Reduce Output Operator [RS_58] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) @@ -3681,7 +3672,7 @@ Stage-0 | | TableScan [TS_48] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 17 [SIMPLE_EDGE] + | |<-Reducer 18 [SIMPLE_EDGE] | Reduce Output Operator [RS_56] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) @@ -3692,7 +3683,7 @@ Stage-0 | | keys:{"0":"_col1 (type: string)","1":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | |<-Map 21 [SIMPLE_EDGE] + | |<-Map 22 [SIMPLE_EDGE] | | Reduce Output Operator [RS_53] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) @@ -3708,7 +3699,7 @@ Stage-0 | | TableScan [TS_46] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 16 [SIMPLE_EDGE] + | |<-Reducer 17 [SIMPLE_EDGE] | Reduce Output Operator [RS_51] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3721,8 +3712,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | |<-Union 15 [SIMPLE_EDGE] - | |<-Map 20 [CONTAINS] + | |<-Union 16 [SIMPLE_EDGE] + | |<-Map 21 [CONTAINS] | | Reduce Output Operator [RS_43] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3741,7 +3732,7 @@ Stage-0 | | TableScan [TS_37] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 14 [CONTAINS] + | |<-Reducer 15 [CONTAINS] | Reduce Output Operator [RS_43] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3755,8 +3746,8 @@ Stage-0 | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 13 [SIMPLE_EDGE] - | |<-Map 12 [CONTAINS] + | |<-Union 14 [SIMPLE_EDGE] + | |<-Map 13 [CONTAINS] | | Reduce Output Operator [RS_34] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3775,7 +3766,7 @@ Stage-0 | | TableScan [TS_26] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 19 [CONTAINS] + | |<-Map 20 [CONTAINS] | Reduce Output Operator [RS_34] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3794,7 +3785,7 @@ Stage-0 | TableScan [TS_28] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 4 [CONTAINS] + |<-Reducer 5 [CONTAINS] Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) @@ -3812,7 +3803,7 @@ Stage-0 | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 11 [SIMPLE_EDGE] + |<-Map 12 [SIMPLE_EDGE] | Reduce Output Operator [RS_23] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) @@ -3828,18 +3819,18 @@ Stage-0 | TableScan [TS_13] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [SIMPLE_EDGE] + |<-Reducer 4 [SIMPLE_EDGE] Reduce Output Operator [RS_21] key expressions:_col2 (type: string) Map-reduce partition columns:_col2 (type: string) sort order:+ Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_159] + Merge Join Operator [MERGEJOIN_159] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 3":"_col1 (type: string)","Map 10":"_col1 (type: string)"} + | keys:{"0":"_col1 (type: string)","1":"_col1 (type: string)"} | outputColumnNames:["_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - |<-Map 10 [BROADCAST_EDGE] + |<-Map 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_18] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) @@ -3855,52 +3846,58 @@ Stage-0 | TableScan [TS_11] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_10] - outputColumnNames:["_col1"] + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) + sort order:+ Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_9] - | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] - | Reduce Output Operator [RS_8] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_7] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_1] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_144] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 9 [CONTAINS] - Reduce Output Operator [RS_8] - key expressions:_col0 (type: string), _col1 (type: string) - Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - sort order:++ - Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_7] - keys:_col0 (type: string), _col1 (type: string) - outputColumnNames:["_col0","_col1"] + Select Operator [SEL_10] + outputColumnNames:["_col1"] + Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_9] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_144] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 10 [CONTAINS] + Reduce Output Operator [RS_8] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_3] + Group By Operator [GBY_7] + keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_145] - predicate:value is not null (type: boolean) + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_3] + outputColumnNames:["_col0","_col1"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_145] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/tez/mrr.q.out ql/src/test/results/clientpositive/tez/mrr.q.out index 2e4e294..4d68a2f 100644 --- ql/src/test/results/clientpositive/tez/mrr.q.out +++ ql/src/test/results/clientpositive/tez/mrr.q.out @@ -1665,9 +1665,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Reducer 2 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1690,7 +1689,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 3 + Map 4 Map Operator Tree: TableScan alias: src @@ -1702,22 +1701,12 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: Group By Operator @@ -1726,13 +1715,23 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 4 + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 1 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: string) diff --git ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out index 5ed3d9e..8b32818 100644 --- ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out +++ ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out @@ -11104,8 +11104,7 @@ STAGE PLANS: Edges: Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) - Map 5 <- Reducer 3 (BROADCAST_EDGE) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -11168,28 +11167,11 @@ STAGE PLANS: Filter Operator predicate: (key = 97) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 '97' (type: string) - 1 '97' (type: string) - outputColumnNames: _col6 - input vertices: - 1 Reducer 3 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Select Operator - expressions: '97' (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: '97' (type: string) + sort order: + + Map-reduce partition columns: '97' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -11201,12 +11183,28 @@ STAGE PLANS: expressions: _col1 (type: string) outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '97' (type: string) - sort order: + - Map-reduce partition columns: '97' (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '97' (type: string) + 1 '97' (type: string) + outputColumnNames: _col6 + input vertices: + 0 Map 5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Select Operator + expressions: '97' (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index cfee6b0..d05e6e3 100644 --- ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -4178,7 +4178,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4193,9 +4193,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4206,25 +4205,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map 2 Map Operator Tree: TableScan alias: srcpart @@ -4244,7 +4228,35 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Reducer 3 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -4259,19 +4271,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -4279,7 +4278,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4479,8 +4478,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4488,12 +4487,32 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: ds (type: string) - sort order: + - Map-reduce partition columns: ds (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map 4 + Map Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 ds (type: string) + 1 ds (type: string) + outputColumnNames: _col8 + input vertices: + 1 Map 3 + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Filter Operator + predicate: (_col8 = '2008-04-08') (type: boolean) + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Map 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4506,30 +4525,6 @@ STAGE PLANS: value expressions: date (type: string) Execution mode: vectorized Reducer 2 - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 ds (type: string) - 1 ds (type: string) - outputColumnNames: _col8 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col8 = '2008-04-08') (type: boolean) - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -4720,9 +4715,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 3 <- Map 2 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4758,13 +4752,27 @@ STAGE PLANS: 0 Map 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE HybridGraceHashJoin: true - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Map-reduce partition columns: '13' (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 1 Map 4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Execution mode: vectorized - Map 3 + Map 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4773,26 +4781,12 @@ STAGE PLANS: Filter Operator predicate: (hr = 13) (type: boolean) Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 '13' (type: string) - 1 '13' (type: string) - input vertices: - 0 Map 2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 + Reduce Output Operator + key expressions: '13' (type: string) + sort order: + + Map-reduce partition columns: '13' (type: string) + Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE + Reducer 3 Execution mode: vectorized Reduce Operator Tree: Group By Operator