diff --git hbase-handler/src/test/results/positive/hbase_queries.q.out hbase-handler/src/test/results/positive/hbase_queries.q.out index 47fc23c..ab06ab7 100644 --- hbase-handler/src/test/results/positive/hbase_queries.q.out +++ hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -118,28 +118,34 @@ STAGE PLANS: TableScan alias: hbase_table_1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -148,11 +154,11 @@ STAGE PLANS: 0 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -167,18 +173,18 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 - Statistics: Num rows: 20 Data size: 4120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 4120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -263,7 +269,7 @@ STAGE PLANS: alias: hbase_table_2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (key < 120) (type: boolean) + predicate: ((key < 120) and key is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -279,7 +285,7 @@ STAGE PLANS: alias: hbase_table_1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (100 < key) (type: boolean) + predicate: ((100 < key) and key is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: int) @@ -481,33 +487,36 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(key) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -521,22 +530,25 @@ STAGE PLANS: TableScan alias: hbase_table_1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) TableScan Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -546,14 +558,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col1} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat diff --git ql/.DS_Store ql/.DS_Store new file mode 100644 index 0000000..ed75636 Binary files /dev/null and ql/.DS_Store differ diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 5fad971..8e17475 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -117,6 +117,9 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { opRules.put(new RuleRegExp("R10", PTFOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getPTFProc()); + opRules.put(new RuleRegExp("R11", + ScriptOperator.getOperatorName() + "%"), + ColumnPrunerProcFactory.getScriptProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 2a8fb2b..cb9e2f8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; @@ -50,6 +51,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; @@ -151,6 +153,68 @@ public static ColumnPrunerGroupByProc getGroupByProc() { return new ColumnPrunerGroupByProc(); } + public static class ColumnPrunerScriptProc implements NodeProcessor { + @SuppressWarnings("unchecked") + public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, + Object... nodeOutputs) throws SemanticException { + + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; + Operator op = (Operator) nd; + RowResolver inputRR = cppCtx.getParseContext().getOpParseCtx().get(op) + .getRowResolver(); + + List prunedCols = cppCtx.getPrunedColList(op.getChildOperators() + .get(0)); + Operator parent = op.getParentOperators().get(0); + RowResolver parentRR = cppCtx.getParseContext().getOpParseCtx() + .get(parent).getRowResolver(); + List sig = parentRR.getRowSchema().getSignature(); + List colList = new ArrayList(); + for (ColumnInfo cI : sig) { + colList.add(cI.getInternalName()); + } + + if (prunedCols.size() != inputRR.getRowSchema().getSignature().size() + && !(op.getChildOperators().get(0) instanceof SelectOperator)) { + ArrayList exprs = new ArrayList(); + ArrayList outputs = new ArrayList(); + Map colExprMap = new HashMap(); + RowResolver outputRS = new RowResolver(); + for (String internalName : prunedCols) { + String[] nm = inputRR.reverseLookup(internalName); + ColumnInfo valueInfo = inputRR.get(nm[0], nm[1]); + ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(), + valueInfo.getInternalName(), nm[0], valueInfo.getIsVirtualCol()); + exprs.add(colDesc); + outputs.add(internalName); + outputRS.put(nm[0], nm[1], + new ColumnInfo(internalName, valueInfo.getType(), nm[0], + valueInfo.getIsVirtualCol(), valueInfo.isHiddenVirtualCol())); + colExprMap.put(internalName, colDesc); + } + SelectDesc select = new SelectDesc(exprs, outputs, false); + + Operator child = op.getChildOperators().get(0); + op.removeChild(child); + SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild( + select, new RowSchema(outputRS.getColumnInfos()), op); + OperatorFactory.makeChild(sel, child); + + OpParseContext parseCtx = new OpParseContext(outputRS); + cppCtx.getParseContext().getOpParseCtx().put(sel, parseCtx); + + sel.setColumnExprMap(colExprMap); + } + + cppCtx.getPrunedColLists().put(op, colList); + return null; + } + } + + public static ColumnPrunerScriptProc getScriptProc() { + return new ColumnPrunerScriptProc(); + } + /** * - Pruning can only be done for Windowing. PTFs are black boxes, * we assume all columns are needed. @@ -159,7 +223,7 @@ public static ColumnPrunerGroupByProc getGroupByProc() { * - finally we set the prunedColList on the ColumnPrunerContx; * and update the RR & signature on the PTFOp. */ - public static class ColumnPrunerPTFProc implements NodeProcessor { + public static class ColumnPrunerPTFProc extends ColumnPrunerScriptProc { public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { @@ -169,16 +233,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. if (!conf.forWindowing()) { - - Operator parent = op.getParentOperators().get(0); - RowResolver parentRR = cppCtx.getParseContext().getOpParseCtx().get(parent).getRowResolver(); - List sig = parentRR.getRowSchema().getSignature(); - List colList = new ArrayList(); - for(ColumnInfo cI : sig) { - colList.add(cI.getInternalName()); - } - cppCtx.getPrunedColLists().put(op, colList); - return null; + return super.process(nd, stack, cppCtx, nodeOutputs); } WindowTableFunctionDef def = (WindowTableFunctionDef) conf.getFuncDef(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6cdaedb..cb284d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2328,6 +2328,57 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input) return output; } + /* + * for inner joins push a 'is not null predicate' to the join sources for + * every non nullSafe predicate. + */ + private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, + QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException { + + if (qb == null || joinTree == null) { + return input; + } + + if (!joinTree.getNoOuterJoin()) { + return input; + } + + if (joinKeys == null || joinKeys.length == 0) { + return input; + } + + ExprNodeDesc filterPred = null; + List nullSafes = joinTree.getNullSafes(); + for (int i = 0; i < joinKeys.length; i++) { + if ( nullSafes.get(i)) { + continue; + } + List args = new ArrayList(); + args.add(joinKeys[i]); + ExprNodeDesc nextExpr = ExprNodeGenericFuncDesc.newInstance( + FunctionRegistry.getFunctionInfo("isnotnull").getGenericUDF(), args); + filterPred = filterPred == null ? nextExpr : ExprNodeDescUtils + .mergePredicates(filterPred, nextExpr); + } + + if (filterPred == null) { + return input; + } + + OpParseContext inputCtx = opParseCtx.get(input); + RowResolver inputRR = inputCtx.getRowResolver(); + + Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( + new FilterDesc(filterPred, false), + new RowSchema(inputRR.getColumnInfos()), input), inputRR); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " + + inputRR.toString()); + } + return output; + } + @SuppressWarnings("nls") private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, ArrayList col_list, @@ -6867,6 +6918,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, for (int i = 0; i < srcOps.length; i++) { // generate a ReduceSink operator for the join String[] srcs = baseSrc[i] != null ? new String[] {baseSrc[i]} : joinTree.getLeftAliases(); + srcOps[i] = genNotNullFilterForJoinSourcePlan(qb, srcOps[i], joinTree, joinKeys[i]); srcOps[i] = genJoinReduceSinkChild(qb, joinKeys[i], srcOps[i], srcs, joinTree.getNextTag()); } diff --git ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out index 6c0f18a..0f51715 100644 --- ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out +++ ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out @@ -103,21 +103,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: a Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -126,14 +132,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -172,13 +178,16 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-1 Map Reduce @@ -186,28 +195,31 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out index 4cb505e..0ef2053 100644 --- ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out +++ ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out @@ -73,13 +73,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-1 Map Reduce @@ -87,28 +90,31 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/allcolref_in_udf.q.out ql/src/test/results/clientpositive/allcolref_in_udf.q.out index be26526..aee81c6 100644 --- ql/src/test/results/clientpositive/allcolref_in_udf.q.out +++ ql/src/test/results/clientpositive/allcolref_in_udf.q.out @@ -84,23 +84,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), value (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key + 1) is not null and (key < 100)) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: (key + 1) (type: double) sort order: + Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) Reduce Operator Tree: Join Operator @@ -110,13 +113,13 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 2 (type: int), concat(_col0, _col1, _col4, _col5) (type: string), concat(_col0, _col1) (type: string), concat(_col4, _col5) (type: string), concat(_col0, _col1, _col4) (type: string), concat(_col0, _col4, _col5) (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE function name: stack Limit Number of rows: 10 diff --git ql/src/test/results/clientpositive/ambiguous_col.q.out ql/src/test/results/clientpositive/ambiguous_col.q.out index 53f6d25..cfd5080 100644 --- ql/src/test/results/clientpositive/ambiguous_col.q.out +++ ql/src/test/results/clientpositive/ambiguous_col.q.out @@ -15,28 +15,34 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -45,14 +51,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -81,27 +87,33 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -110,14 +122,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -146,27 +158,33 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -175,14 +193,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/annotate_stats_join.q.out ql/src/test/results/clientpositive/annotate_stats_join.q.out index 127b73a..bd91ded 100644 --- ql/src/test/results/clientpositive/annotate_stats_join.q.out +++ ql/src/test/results/clientpositive/annotate_stats_join.q.out @@ -243,24 +243,32 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: deptname (type: string) + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: deptname (type: string) TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: lastname (type: string) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: lastname (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -368,17 +376,17 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 12 Data size: 2196 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1464 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 2196 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1464 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 12 Data size: 2196 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1464 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -468,35 +476,47 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: deptname (type: string) + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: deptname (type: string) TableScan alias: e1 Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 2 - value expressions: lastname (type: string) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 2 + value expressions: lastname (type: string) TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: lastname (type: string) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: lastname (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -606,17 +626,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {VALUE._col0} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 72 Data size: 19728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 8768 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string), _col8 (type: string), _col9 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 72 Data size: 19728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 8768 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 72 Data size: 19728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 8768 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -704,35 +724,47 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: deptname (type: string) + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: deptname (type: string) TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int) - sort order: + - Map-reduce partition columns: deptid (type: int) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: lastname (type: string) + Filter Operator + isSamplingPred: false + predicate: deptid is not null (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int) + sort order: + + Map-reduce partition columns: deptid (type: int) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: lastname (type: string) TableScan alias: l Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: locid (type: int) - sort order: + - Map-reduce partition columns: locid (type: int) - Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE - tag: 2 - value expressions: state (type: string), zip (type: bigint), year (type: int) + Filter Operator + isSamplingPred: false + predicate: locid is not null (type: boolean) + Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: locid (type: int) + sort order: + + Map-reduce partition columns: locid (type: int) + Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE + tag: 2 + value expressions: state (type: string), zip (type: bigint), year (type: int) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -889,17 +921,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col10, _col11 - Statistics: Num rows: 96 Data size: 26964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 15724 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: bigint), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 96 Data size: 26964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 15724 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 96 Data size: 26964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 56 Data size: 15724 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -989,35 +1021,47 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: UDFToDouble(deptid) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(deptid) (type: double) - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: deptid (type: int), deptname (type: string) + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(deptid) is not null (type: boolean) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(deptid) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(deptid) (type: double) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: deptid (type: int), deptname (type: string) TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: UDFToDouble(deptid) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(deptid) (type: double) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: lastname (type: string), deptid (type: int) + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(deptid) is not null (type: boolean) + Statistics: Num rows: 3 Data size: 281 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(deptid) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(deptid) (type: double) + Statistics: Num rows: 3 Data size: 281 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: lastname (type: string), deptid (type: int) TableScan alias: l Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: UDFToDouble(state) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(state) (type: double) - Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE - tag: 2 - value expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(state) is not null (type: boolean) + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(state) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(state) (type: double) + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + tag: 2 + value expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1174,17 +1218,17 @@ STAGE PLANS: 1 {VALUE._col0} {VALUE._col1} 2 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col10, _col11 - Statistics: Num rows: 4 Data size: 1156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: bigint), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 4 Data size: 1156 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1270,22 +1314,30 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int), deptname (type: string) - sort order: ++ - Map-reduce partition columns: deptid (type: int), deptname (type: string) + Filter Operator + isSamplingPred: false + predicate: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 + Reduce Output Operator + key expressions: deptid (type: int), deptname (type: string) + sort order: ++ + Map-reduce partition columns: deptid (type: int), deptname (type: string) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int), lastname (type: string) - sort order: ++ - Map-reduce partition columns: deptid (type: int), lastname (type: string) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: (deptid is not null and lastname is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int), lastname (type: string) + sort order: ++ + Map-reduce partition columns: deptid (type: int), lastname (type: string) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1393,17 +1445,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1513,33 +1565,45 @@ STAGE PLANS: alias: d Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int), deptname (type: string) - sort order: ++ - Map-reduce partition columns: deptid (type: int), deptname (type: string) + Filter Operator + isSamplingPred: false + predicate: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 + Reduce Output Operator + key expressions: deptid (type: int), deptname (type: string) + sort order: ++ + Map-reduce partition columns: deptid (type: int), deptname (type: string) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 TableScan alias: e Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: deptid (type: int), lastname (type: string) - sort order: ++ - Map-reduce partition columns: deptid (type: int), lastname (type: string) - Statistics: Num rows: 6 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: (deptid is not null and lastname is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: deptid (type: int), lastname (type: string) + sort order: ++ + Map-reduce partition columns: deptid (type: int), lastname (type: string) + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 TableScan alias: l Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Reduce Output Operator - key expressions: locid (type: int), state (type: string) - sort order: ++ - Map-reduce partition columns: locid (type: int), state (type: string) - Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE - tag: 2 - value expressions: zip (type: bigint), year (type: int) + Filter Operator + isSamplingPred: false + predicate: (locid is not null and state is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: locid (type: int), state (type: string) + sort order: ++ + Map-reduce partition columns: locid (type: int), state (type: string) + Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + tag: 2 + value expressions: zip (type: bigint), year (type: int) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1696,17 +1760,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 2 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col10, _col11 - Statistics: Num rows: 48 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: bigint), _col11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 48 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 48 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/auto_join1.q.out ql/src/test/results/clientpositive/auto_join1.q.out index 0e1a610..13a2d6c 100644 --- ql/src/test/results/clientpositive/auto_join1.q.out +++ ql/src/test/results/clientpositive/auto_join1.q.out @@ -31,13 +31,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -45,29 +48,32 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join10.q.out ql/src/test/results/clientpositive/auto_join10.q.out index 8e61c32..8bb19a8 100644 --- ql/src/test/results/clientpositive/auto_join10.q.out +++ ql/src/test/results/clientpositive/auto_join10.q.out @@ -31,17 +31,20 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {_col0} {_col1} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-2 Map Reduce @@ -49,34 +52,37 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {_col0} {_col1} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: string), _col3 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(hash(_col2,_col3)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2,_col3)) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/auto_join11.q.out ql/src/test/results/clientpositive/auto_join11.q.out index 9407838..1cb2cce 100644 --- ql/src/test/results/clientpositive/auto_join11.q.out +++ ql/src/test/results/clientpositive/auto_join11.q.out @@ -32,12 +32,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -53,12 +53,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -69,11 +69,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git ql/src/test/results/clientpositive/auto_join12.q.out ql/src/test/results/clientpositive/auto_join12.q.out index 3866659..6c5208e 100644 --- ql/src/test/results/clientpositive/auto_join12.q.out +++ ql/src/test/results/clientpositive/auto_join12.q.out @@ -41,12 +41,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 80) and (key < 100)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -61,12 +61,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -84,12 +84,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -103,11 +103,11 @@ STAGE PLANS: 1 _col0 (type: string) 2 _col0 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git ql/src/test/results/clientpositive/auto_join13.q.out ql/src/test/results/clientpositive/auto_join13.q.out index 61cdda0..00ce62f 100644 --- ql/src/test/results/clientpositive/auto_join13.q.out +++ ql/src/test/results/clientpositive/auto_join13.q.out @@ -41,12 +41,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -59,12 +59,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 200) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {_col0} {_col3} @@ -80,12 +80,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -96,31 +96,34 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col3} - 1 - keys: - 0 (_col0 + _col2) (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 22 Data size: 2302 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 + _col2) is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col3} + 1 + keys: + 0 (_col0 + _col2) (type: double) + 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col3 - Statistics: Num rows: 22 Data size: 2302 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(hash(_col0,_col3)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col0,_col3)) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index ad8760f..f5a110a 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -36,8 +36,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -53,8 +53,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -65,14 +65,14 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col5 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/auto_join15.q.out ql/src/test/results/clientpositive/auto_join15.q.out index 60b0a1d..f1f3f64 100644 --- ql/src/test/results/clientpositive/auto_join15.q.out +++ ql/src/test/results/clientpositive/auto_join15.q.out @@ -32,13 +32,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -46,32 +49,35 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - sort order: ++++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col1,_col2,_col3)) mode: hash diff --git ql/src/test/results/clientpositive/auto_join16.q.out ql/src/test/results/clientpositive/auto_join16.q.out index 2880a2a..32965bf 100644 --- ql/src/test/results/clientpositive/auto_join16.q.out +++ ql/src/test/results/clientpositive/auto_join16.q.out @@ -32,12 +32,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and (key > 20)) and (value < 200)) (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -53,8 +53,8 @@ STAGE PLANS: alias: tab Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 20) and (value < 200)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (value < 200)) (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -65,11 +65,11 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 key (type: string), value (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 440 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 440 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git ql/src/test/results/clientpositive/auto_join17.q.out ql/src/test/results/clientpositive/auto_join17.q.out index 869c3b6..3c68f5d 100644 --- ql/src/test/results/clientpositive/auto_join17.q.out +++ ql/src/test/results/clientpositive/auto_join17.q.out @@ -31,13 +31,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -45,29 +48,32 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col4) (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col4) (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join19.q.out ql/src/test/results/clientpositive/auto_join19.q.out index c266dd6..eab7182 100644 --- ql/src/test/results/clientpositive/auto_join19.q.out +++ ql/src/test/results/clientpositive/auto_join19.q.out @@ -33,13 +33,16 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -47,29 +50,32 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col7 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col7 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join2.q.out ql/src/test/results/clientpositive/auto_join2.q.out index fda4daf..4e8a706 100644 --- ql/src/test/results/clientpositive/auto_join2.q.out +++ ql/src/test/results/clientpositive/auto_join2.q.out @@ -34,24 +34,30 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {key} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) src3 TableScan alias: src3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 (_col0 + _col4) (type: double) - 1 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 (_col0 + _col4) (type: double) + 1 UDFToDouble(key) (type: double) Stage: Stage-6 Map Reduce @@ -59,40 +65,46 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {key} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} - 1 {value} + 0 {key} + 1 {key} keys: - 0 (_col0 + _col4) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col9 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col4 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 + _col4) is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1649 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 (_col0 + _col4) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col9 + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join22.q.out ql/src/test/results/clientpositive/auto_join22.q.out index d85806f..9a85118 100644 --- ql/src/test/results/clientpositive/auto_join22.q.out +++ ql/src/test/results/clientpositive/auto_join22.q.out @@ -24,24 +24,30 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) src5:src4 TableScan alias: src4 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {_col3} - keys: - 0 key (type: string) - 1 _col2 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {_col3} + keys: + 0 key (type: string) + 1 _col2 (type: string) Stage: Stage-3 Map Reduce @@ -49,45 +55,48 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {_col3} - keys: - 0 key (type: string) - 1 _col2 (type: string) - outputColumnNames: _col7 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col7 (type: string) - outputColumnNames: _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(hash(_col3)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {_col3} + keys: + 0 key (type: string) + 1 _col2 (type: string) + outputColumnNames: _col7 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string) + outputColumnNames: _col3 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col3)) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/auto_join24.q.out ql/src/test/results/clientpositive/auto_join24.q.out index cdb9520..33c98cd 100644 --- ql/src/test/results/clientpositive/auto_join24.q.out +++ ql/src/test/results/clientpositive/auto_join24.q.out @@ -40,13 +40,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {cnt} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {cnt} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -54,30 +57,33 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {cnt} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 339 Data size: 1630 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {cnt} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 339 Data size: 1630 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(_col1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col1) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/auto_join26.q.out ql/src/test/results/clientpositive/auto_join26.q.out index d55de11..37fe6df 100644 --- ql/src/test/results/clientpositive/auto_join26.q.out +++ ql/src/test/results/clientpositive/auto_join26.q.out @@ -35,13 +35,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -49,33 +52,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -84,14 +90,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/auto_join27.q.out ql/src/test/results/clientpositive/auto_join27.q.out index e2b1b4d..27d8328 100644 --- ql/src/test/results/clientpositive/auto_join27.q.out +++ ql/src/test/results/clientpositive/auto_join27.q.out @@ -37,32 +37,32 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 200) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -82,12 +82,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 200) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -101,7 +101,7 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 23 Data size: 2704 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 36 Data size: 4308 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -111,9 +111,9 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 25 Data size: 2974 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 4738 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 25 Data size: 2974 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 4738 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash @@ -129,14 +129,14 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 200) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 23 Data size: 2704 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 36 Data size: 4308 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -146,9 +146,9 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 25 Data size: 2974 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 4738 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 25 Data size: 2974 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 4738 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash diff --git ql/src/test/results/clientpositive/auto_join29.q.out ql/src/test/results/clientpositive/auto_join29.q.out index d9a0c1a..ec76217 100644 --- ql/src/test/results/clientpositive/auto_join29.q.out +++ ql/src/test/results/clientpositive/auto_join29.q.out @@ -3551,8 +3551,8 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -3567,8 +3567,8 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 10) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -3586,8 +3586,8 @@ STAGE PLANS: alias: src3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3601,25 +3601,25 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 6 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ - Statistics: Num rows: 6 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/auto_join3.q.out ql/src/test/results/clientpositive/auto_join3.q.out index e6e1b78..0c58f4c 100644 --- ql/src/test/results/clientpositive/auto_join3.q.out +++ ql/src/test/results/clientpositive/auto_join3.q.out @@ -34,28 +34,34 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) src2 TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-5 Map Reduce @@ -63,32 +69,35 @@ STAGE PLANS: TableScan alias: src3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key} - 1 - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col9 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col9 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join30.q.out ql/src/test/results/clientpositive/auto_join30.q.out index 2b9185c..283eeff 100644 --- ql/src/test/results/clientpositive/auto_join30.q.out +++ ql/src/test/results/clientpositive/auto_join30.q.out @@ -33,19 +33,22 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -184,13 +187,13 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -200,11 +203,11 @@ STAGE PLANS: 0 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash @@ -223,20 +226,23 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -757,20 +763,23 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1013,19 +1022,19 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -1037,11 +1046,11 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 outputColumnNames: _col2, _col3 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) mode: hash @@ -1060,19 +1069,22 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1086,20 +1098,23 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/auto_join32.q.out ql/src/test/results/clientpositive/auto_join32.q.out index 0b21d1e..2267218 100644 --- ql/src/test/results/clientpositive/auto_join32.q.out +++ ql/src/test/results/clientpositive/auto_join32.q.out @@ -41,13 +41,16 @@ STAGE PLANS: TableScan alias: s Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {registration} - keys: - 0 name (type: string) - 1 name (type: string) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {registration} + keys: + 0 name (type: string) + 1 name (type: string) Stage: Stage-2 Map Reduce @@ -55,32 +58,35 @@ STAGE PLANS: TableScan alias: v Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {name} - 1 {registration} - keys: - 0 name (type: string) - 1 name (type: string) - outputColumnNames: _col0, _col7 + Filter Operator + predicate: name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col7 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {name} + 1 {registration} + keys: + 0 name (type: string) + 1 name (type: string) outputColumnNames: _col0, _col7 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - aggregations: count(DISTINCT _col7) - keys: _col0 (type: string), _col7 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col7 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count(DISTINCT _col7) + keys: _col0 (type: string), _col7 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -161,28 +167,31 @@ STAGE PLANS: TableScan alias: s Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {name} - 1 {registration} - keys: - 0 name (type: string) - 1 name (type: string) - outputColumnNames: _col0, _col7 - Select Operator - expressions: _col0 (type: string), _col7 (type: string) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {name} + 1 {registration} + keys: + 0 name (type: string) + 1 name (type: string) outputColumnNames: _col0, _col7 - Group By Operator - aggregations: count(DISTINCT _col7) - keys: _col0 (type: string), _col7 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col7 + Group By Operator + aggregations: count(DISTINCT _col7) + keys: _col0 (type: string), _col7 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col1:0._col0) @@ -274,28 +283,31 @@ STAGE PLANS: TableScan alias: s Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {name} - 1 {registration} - keys: - 0 name (type: string) - 1 name (type: string) - outputColumnNames: _col0, _col7 - Select Operator - expressions: _col0 (type: string), _col7 (type: string) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {name} + 1 {registration} + keys: + 0 name (type: string) + 1 name (type: string) outputColumnNames: _col0, _col7 - Group By Operator - aggregations: count(DISTINCT _col7) - keys: _col0 (type: string), _col7 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col7 + Group By Operator + aggregations: count(DISTINCT _col7) + keys: _col0 (type: string), _col7 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col1:0._col0) diff --git ql/src/test/results/clientpositive/auto_join9.q.out ql/src/test/results/clientpositive/auto_join9.q.out index 3edeaed..63762df 100644 --- ql/src/test/results/clientpositive/auto_join9.q.out +++ ql/src/test/results/clientpositive/auto_join9.q.out @@ -31,13 +31,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -45,29 +48,32 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col7 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col7 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/auto_join_reordering_values.q.out ql/src/test/results/clientpositive/auto_join_reordering_values.q.out index b440420..3f4f902 100644 --- ql/src/test/results/clientpositive/auto_join_reordering_values.q.out +++ ql/src/test/results/clientpositive/auto_join_reordering_values.q.out @@ -179,23 +179,31 @@ STAGE PLANS: alias: dim_pay_date Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: date (type: string) - sort order: + - Map-reduce partition columns: date (type: string) + Filter Operator + isSamplingPred: false + predicate: date is not null (type: boolean) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - tag: 1 + Reduce Output Operator + key expressions: date (type: string) + sort order: + + Map-reduce partition columns: date (type: string) + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + tag: 1 TableScan alias: orderpayment Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: date (type: string) - sort order: + - Map-reduce partition columns: date (type: string) + Filter Operator + isSamplingPred: false + predicate: (((date is not null and dealid is not null) and cityid is not null) and userid is not null) (type: boolean) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: dealid (type: int), cityid (type: int), userid (type: int) + Reduce Output Operator + key expressions: date (type: string) + sort order: + + Map-reduce partition columns: date (type: string) + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: dealid (type: int), cityid (type: int), userid (type: int) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -289,12 +297,16 @@ STAGE PLANS: alias: deal Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: dealid (type: int) - sort order: + - Map-reduce partition columns: dealid (type: int) + Filter Operator + isSamplingPred: false + predicate: dealid is not null (type: boolean) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - tag: 1 + Reduce Output Operator + key expressions: dealid (type: int) + sort order: + + Map-reduce partition columns: dealid (type: int) + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + tag: 1 Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -409,12 +421,16 @@ STAGE PLANS: alias: order_city Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: cityid (type: int) - sort order: + - Map-reduce partition columns: cityid (type: int) + Filter Operator + isSamplingPred: false + predicate: cityid is not null (type: boolean) Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE - tag: 1 + Reduce Output Operator + key expressions: cityid (type: int) + sort order: + + Map-reduce partition columns: cityid (type: int) + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + tag: 1 Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -529,12 +545,16 @@ STAGE PLANS: alias: user Statistics: Num rows: 100 Data size: 288 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: userid (type: int) - sort order: + - Map-reduce partition columns: userid (type: int) - Statistics: Num rows: 100 Data size: 288 Basic stats: COMPLETE Column stats: NONE - tag: 1 + Filter Operator + isSamplingPred: false + predicate: userid is not null (type: boolean) + Statistics: Num rows: 50 Data size: 144 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: userid (type: int) + sort order: + + Map-reduce partition columns: userid (type: int) + Statistics: Num rows: 50 Data size: 144 Basic stats: COMPLETE Column stats: NONE + tag: 1 Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -614,11 +634,11 @@ STAGE PLANS: 0 {VALUE._col7} {VALUE._col13} 1 outputColumnNames: _col8, _col14 - Statistics: Num rows: 110 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 158 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col8 (type: string), _col14 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 110 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 158 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 5 Statistics: Num rows: 5 Data size: 10 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/auto_join_without_localtask.q.out ql/src/test/results/clientpositive/auto_join_without_localtask.q.out index 01805ed..d2efd8f 100644 --- ql/src/test/results/clientpositive/auto_join_without_localtask.q.out +++ ql/src/test/results/clientpositive/auto_join_without_localtask.q.out @@ -5,9 +5,11 @@ POSTHOOK: query: explain select a.* from src a join src b on a.key=b.key limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 depends on stages: Stage-3, Stage-4, Stage-1 @@ -15,79 +17,107 @@ STAGE PLANS: Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Limit - Number of rows: 40 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b + + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Limit - Number of rows: 40 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Stage: Stage-1 Map Reduce @@ -95,20 +125,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -117,17 +153,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 40 - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -148,7 +184,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-5:CONDITIONAL -RUN: Stage-3:MAPRED +RUN: Stage-7:MAPREDLOCAL +RUN: Stage-4:MAPRED 238 val_238 238 val_238 86 val_86 @@ -196,13 +233,16 @@ POSTHOOK: query: explain select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-10 is a root stage , consists of Stage-8, Stage-9, Stage-1 - Stage-8 has a backup stage: Stage-1 - Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-5, Stage-6, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-10 is a root stage , consists of Stage-13, Stage-14, Stage-1 + Stage-13 has a backup stage: Stage-1 + Stage-8 depends on stages: Stage-13 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-11, Stage-6, Stage-2 + Stage-11 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-11 Stage-6 has a backup stage: Stage-2 Stage-2 - Stage-9 has a backup stage: Stage-1 + Stage-14 has a backup stage: Stage-1 + Stage-9 depends on stages: Stage-14 Stage-1 Stage-0 depends on stages: Stage-5, Stage-6, Stage-2 @@ -210,41 +250,75 @@ STAGE PLANS: Stage: Stage-10 Conditional Operator + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Stage: Stage-7 Conditional Operator + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + c + TableScan + alias: c + Filter Operator + predicate: value is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -272,41 +346,35 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - c - TableScan - alias: c Stage: Stage-6 Map Reduce Map Operator Tree: TableScan alias: c - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1 - Limit - Number of rows: 40 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work Alias -> Map Local Tables: @@ -325,16 +393,19 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -343,53 +414,67 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 40 - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-14 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Stage: Stage-1 Map Reduce @@ -397,20 +482,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -419,7 +510,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -442,8 +533,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-10:CONDITIONAL -RUN: Stage-8:MAPRED +RUN: Stage-14:MAPREDLOCAL +RUN: Stage-9:MAPRED RUN: Stage-7:CONDITIONAL +RUN: Stage-11:MAPREDLOCAL RUN: Stage-5:MAPRED 238 val_238 238 val_238 @@ -495,8 +588,9 @@ STAGE DEPENDENCIES: Stage-10 is a root stage , consists of Stage-13, Stage-14, Stage-1 Stage-13 has a backup stage: Stage-1 Stage-8 depends on stages: Stage-13 - Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-5, Stage-6, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-11, Stage-6, Stage-2 + Stage-11 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-11 Stage-6 has a backup stage: Stage-2 Stage-2 Stage-14 has a backup stage: Stage-1 @@ -519,7 +613,7 @@ STAGE PLANS: TableScan alias: b Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (key is not null and (key > 100)) (type: boolean) HashTable Sink Operator condition expressions: 0 {key} {value} @@ -534,7 +628,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key > 100) (type: boolean) + predicate: ((key is not null and value is not null) and (key > 100)) (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 @@ -557,6 +651,26 @@ STAGE PLANS: Stage: Stage-7 Conditional Operator + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + c + TableScan + alias: c + Filter Operator + predicate: value is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -584,41 +698,35 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - c - TableScan - alias: c Stage: Stage-6 Map Reduce Map Operator Tree: TableScan alias: c - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1 - Limit - Number of rows: 40 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 40 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work Alias -> Map Local Tables: @@ -637,16 +745,19 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -655,17 +766,17 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 40 - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 40 Data size: 4040 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -682,7 +793,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key > 100) (type: boolean) + predicate: ((key is not null and value is not null) and (key > 100)) (type: boolean) HashTable Sink Operator condition expressions: 0 {value} @@ -697,7 +808,7 @@ STAGE PLANS: TableScan alias: b Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (key is not null and (key > 100)) (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 @@ -724,24 +835,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key > 100)) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (key > 100)) (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -751,7 +862,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -777,6 +888,7 @@ RUN: Stage-10:CONDITIONAL RUN: Stage-14:MAPREDLOCAL RUN: Stage-9:MAPRED RUN: Stage-7:CONDITIONAL +RUN: Stage-11:MAPREDLOCAL RUN: Stage-5:MAPRED 238 val_238 238 val_238 @@ -835,17 +947,17 @@ Logs: #### A masked pattern was here #### FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-5 + Stage-11 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: -- fallback to common join select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40 @@ -856,7 +968,7 @@ RUN: Stage-10:CONDITIONAL RUN: Stage-14:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-7:CONDITIONAL -RUN: Stage-5:MAPRED +RUN: Stage-11:MAPREDLOCAL RUN: Stage-2:MAPRED 103 val_103 103 val_103 diff --git ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out index d49ebdd..7463991 100644 --- ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out @@ -63,23 +63,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -150,29 +153,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -295,29 +301,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -373,29 +382,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -492,12 +504,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -595,7 +607,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -722,7 +734,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -839,12 +851,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 8) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -936,11 +948,14 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE @@ -948,11 +963,14 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -960,9 +978,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1052,12 +1070,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1147,12 +1165,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1261,7 +1279,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -1392,39 +1410,42 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-8 Conditional Operator @@ -1667,42 +1688,45 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash + Select Operator + expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out index 3ab9b94..72640df 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out @@ -151,25 +151,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -373,25 +377,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -654,14 +662,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -669,25 +680,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -975,14 +989,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -990,25 +1007,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1196,25 +1216,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out index ffad9d4..3b791af 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out @@ -81,12 +81,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -102,18 +102,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -123,9 +123,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -141,18 +141,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -162,9 +162,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -270,24 +270,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() bucketGroup: true keys: key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -319,12 +319,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -346,9 +346,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out index 07614e5..c660cd0 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out @@ -200,14 +200,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-2 Map Reduce @@ -216,29 +220,33 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -553,14 +561,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-2 Map Reduce @@ -569,29 +581,33 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -904,14 +920,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt]} @@ -927,30 +947,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1325,31 +1349,39 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 b TableScan alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt]} @@ -1366,33 +1398,37 @@ STAGE PLANS: alias: c Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out index ef2b72a..4abda32 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out @@ -135,7 +135,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -368,31 +368,39 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 b TableScan alias: b Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 d TableScan alias: d @@ -414,43 +422,47 @@ STAGE PLANS: alias: c Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 + Inner Join 1 to 2 condition expressions: 0 1 + 2 keys: - 0 - 1 - Position of Big Table: 0 - Statistics: Num rows: 280 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 280 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 + 1 + Position of Big Table: 0 + Statistics: Num rows: 139 Data size: 14064 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 139 Data size: 14064 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -690,7 +702,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out index 93ba35b..ab82fea 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out @@ -82,39 +82,42 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 - Select Operator - expressions: _col1 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: _col0 (type: int), _col2 (type: int) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Select Operator + expressions: _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-0 Move Operator @@ -260,39 +263,42 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 Select Operator - expressions: _col1 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: _col0 (type: int), _col2 (type: int) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Select Operator + expressions: _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-0 Move Operator @@ -438,39 +444,42 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 Select Operator - expressions: _col1 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: _col0 (type: int), _col2 (type: int) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Select Operator + expressions: _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out index 9985d81..52a3194 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out @@ -131,25 +131,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -414,14 +418,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -429,25 +436,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -735,14 +745,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -750,25 +763,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -956,25 +972,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out index 2402fc2..d807791 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out @@ -131,25 +131,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -304,25 +308,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -582,14 +590,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -597,25 +608,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -854,14 +868,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -869,25 +886,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1075,25 +1095,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out index 39e3992..35e0a30 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out @@ -147,25 +147,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -320,25 +324,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -598,14 +606,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -613,25 +624,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -870,14 +884,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -885,25 +902,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1091,25 +1111,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out index 2f79ac0..af3d9d6 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out @@ -112,25 +112,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -279,25 +283,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -458,14 +466,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -473,25 +484,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -628,14 +642,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -643,25 +660,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -794,25 +814,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out index ba9a54e..2dae094 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out @@ -96,28 +96,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -200,28 +206,34 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -304,28 +316,34 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) TableScan alias: h Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -421,56 +439,62 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -513,56 +537,62 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-6 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -579,56 +609,62 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-7 Map Reduce Map Operator Tree: TableScan alias: c - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -638,27 +674,36 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -668,9 +713,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -719,26 +764,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -791,28 +839,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -891,28 +945,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1008,56 +1068,62 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1100,56 +1166,62 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-6 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1166,56 +1238,62 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) Stage: Stage-7 Map Reduce Map Operator Tree: TableScan alias: c - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1225,27 +1303,36 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1255,9 +1342,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1323,53 +1410,59 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1400,53 +1493,59 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) c TableScan alias: c - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1477,53 +1576,59 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) Stage: Stage-6 Map Reduce Map Operator Tree: TableScan alias: c - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1547,26 +1652,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1619,28 +1727,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out index c8d1b42..05ef5d8 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out @@ -164,25 +164,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -388,25 +392,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -717,14 +725,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -732,25 +743,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1085,14 +1099,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -1100,25 +1117,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1354,25 +1374,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out index 975e903..e423d14 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out @@ -164,25 +164,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -388,25 +392,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -719,14 +727,17 @@ STAGE PLANS: TableScan alias: b GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-3 Map Reduce @@ -734,25 +745,28 @@ STAGE PLANS: TableScan alias: a GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1087,14 +1101,17 @@ STAGE PLANS: TableScan alias: a GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-4 Map Reduce @@ -1102,25 +1119,28 @@ STAGE PLANS: TableScan alias: b GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1356,25 +1376,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out index c8ecb8c..2a01f4b 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out @@ -63,23 +63,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -143,29 +146,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -246,29 +252,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -396,29 +405,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -563,29 +575,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -682,12 +697,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -785,7 +800,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -912,7 +927,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -1029,12 +1044,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 8) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1131,13 +1146,16 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-2 Map Reduce @@ -1149,27 +1167,30 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1241,12 +1262,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1327,23 +1348,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1418,12 +1442,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1532,7 +1556,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -1642,36 +1666,40 @@ STAGE PLANS: subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-3 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1699,36 +1727,40 @@ STAGE PLANS: subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1752,23 +1784,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1844,42 +1879,46 @@ STAGE PLANS: subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-3 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1908,42 +1947,46 @@ STAGE PLANS: subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1968,29 +2011,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2083,42 +2129,46 @@ STAGE PLANS: subq2:subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2171,42 +2221,46 @@ STAGE PLANS: subq2:subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2233,29 +2287,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2378,42 +2435,46 @@ STAGE PLANS: src1:subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-9 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2564,42 +2625,46 @@ STAGE PLANS: src1:subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-10 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2621,32 +2686,35 @@ STAGE PLANS: Stage: Stage-1 Map Reduce Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2676,42 +2744,46 @@ STAGE PLANS: src2:subq2:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-12 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2740,42 +2812,46 @@ STAGE PLANS: src2:subq2:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-13 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2800,29 +2876,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2931,7 +3010,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -2949,7 +3028,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -2998,7 +3077,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3016,7 +3095,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3061,12 +3140,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -3175,13 +3254,15 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) Stage: Stage-3 Map Reduce @@ -3189,7 +3270,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3238,7 +3319,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3255,23 +3336,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -3296,7 +3379,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -3435,7 +3518,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3453,7 +3536,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3502,7 +3585,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3520,7 +3603,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3565,7 +3648,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -3694,7 +3777,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 8) (type: boolean) + predicate: ((key < 8) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3712,7 +3795,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 8) (type: boolean) + predicate: ((key < 8) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3761,7 +3844,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 8) (type: boolean) + predicate: ((key < 8) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3779,7 +3862,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 8) (type: boolean) + predicate: ((key < 8) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3824,12 +3907,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 8) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -3924,13 +4007,15 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) Stage: Stage-3 Map Reduce @@ -3938,7 +4023,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3987,7 +4072,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4004,23 +4089,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -4045,12 +4132,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -4144,7 +4231,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4161,23 +4248,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -4205,13 +4294,15 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) Stage: Stage-4 Map Reduce @@ -4219,7 +4310,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4263,23 +4354,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -4371,7 +4465,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4388,7 +4482,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4408,7 +4502,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4463,7 +4557,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4480,7 +4574,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4500,7 +4594,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4555,7 +4649,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4572,7 +4666,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4592,7 +4686,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: (key < 6) (type: boolean) + predicate: ((key < 6) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4640,12 +4734,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -4765,13 +4859,15 @@ STAGE PLANS: a:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) Stage: Stage-3 Map Reduce @@ -4779,7 +4875,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4828,7 +4924,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4845,23 +4941,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -4886,7 +4984,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) diff --git ql/src/test/results/clientpositive/bucket_map_join_1.q.out ql/src/test/results/clientpositive/bucket_map_join_1.q.out index aa2a4b9..8674a6c 100644 --- ql/src/test/results/clientpositive/bucket_map_join_1.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_1.q.out @@ -119,14 +119,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -135,29 +139,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 - Statistics: Num rows: 0 Data size: 22 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 0 Data size: 22 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucket_map_join_2.q.out ql/src/test/results/clientpositive/bucket_map_join_2.q.out index 77602bb..8a5984d 100644 --- ql/src/test/results/clientpositive/bucket_map_join_2.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_2.q.out @@ -119,14 +119,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -135,29 +139,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 - Statistics: Num rows: 0 Data size: 22 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 0 Data size: 22 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketcontext_1.q.out ql/src/test/results/clientpositive/bucketcontext_1.q.out index 3fe74dc..1513515 100644 --- ql/src/test/results/clientpositive/bucketcontext_1.q.out +++ ql/src/test/results/clientpositive/bucketcontext_1.q.out @@ -203,14 +203,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt]} @@ -226,30 +230,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -463,25 +471,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_2.q.out ql/src/test/results/clientpositive/bucketcontext_2.q.out index 9252071..d18a9be 100644 --- ql/src/test/results/clientpositive/bucketcontext_2.q.out +++ ql/src/test/results/clientpositive/bucketcontext_2.q.out @@ -187,14 +187,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt]} @@ -210,30 +214,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -447,25 +455,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_3.q.out ql/src/test/results/clientpositive/bucketcontext_3.q.out index daddad3..e12c155 100644 --- ql/src/test/results/clientpositive/bucketcontext_3.q.out +++ ql/src/test/results/clientpositive/bucketcontext_3.q.out @@ -233,14 +233,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} @@ -256,30 +260,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -444,25 +452,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_4.q.out ql/src/test/results/clientpositive/bucketcontext_4.q.out index 40ded66..77b4882 100644 --- ql/src/test/results/clientpositive/bucketcontext_4.q.out +++ ql/src/test/results/clientpositive/bucketcontext_4.q.out @@ -249,14 +249,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} @@ -272,30 +276,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -460,25 +468,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_5.q.out ql/src/test/results/clientpositive/bucketcontext_5.q.out index a67c4c5..fa1cfc5 100644 --- ql/src/test/results/clientpositive/bucketcontext_5.q.out +++ ql/src/test/results/clientpositive/bucketcontext_5.q.out @@ -121,14 +121,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} @@ -144,30 +148,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -326,25 +334,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_6.q.out ql/src/test/results/clientpositive/bucketcontext_6.q.out index c08ecdc..aac66f8 100644 --- ql/src/test/results/clientpositive/bucketcontext_6.q.out +++ ql/src/test/results/clientpositive/bucketcontext_6.q.out @@ -139,14 +139,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} @@ -162,30 +166,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -397,25 +405,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_7.q.out ql/src/test/results/clientpositive/bucketcontext_7.q.out index 810f736..78c4f94 100644 --- ql/src/test/results/clientpositive/bucketcontext_7.q.out +++ ql/src/test/results/clientpositive/bucketcontext_7.q.out @@ -266,14 +266,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} @@ -289,30 +293,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -528,25 +536,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketcontext_8.q.out ql/src/test/results/clientpositive/bucketcontext_8.q.out index 3d534ce..ad7fec9 100644 --- ql/src/test/results/clientpositive/bucketcontext_8.q.out +++ ql/src/test/results/clientpositive/bucketcontext_8.q.out @@ -266,14 +266,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} @@ -289,30 +293,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -528,25 +536,29 @@ STAGE PLANS: alias: b Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/bucketmapjoin1.q.out ql/src/test/results/clientpositive/bucketmapjoin1.q.out index 840c959..10f1af4 100644 --- ql/src/test/results/clientpositive/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin1.q.out @@ -107,7 +107,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (ds = '2008-04-08') (type: boolean) + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE HashTable Sink Operator condition expressions: @@ -232,14 +232,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {} @@ -497,14 +501,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} @@ -520,49 +528,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 0 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -990,14 +1002,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} @@ -1013,54 +1029,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin10.q.out ql/src/test/results/clientpositive/bucketmapjoin10.q.out index 29bd7e2..88ecf40 100644 --- ql/src/test/results/clientpositive/bucketmapjoin10.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin10.q.out @@ -298,14 +298,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 1737 Data size: 6950 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 869 Data size: 3477 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -314,29 +318,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 1737 Data size: 6950 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 1910 Data size: 7645 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1910 Data size: 7645 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 869 Data size: 3477 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 955 Data size: 3824 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 955 Data size: 3824 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin11.q.out ql/src/test/results/clientpositive/bucketmapjoin11.q.out index 19d1178..4ee1fa0 100644 --- ql/src/test/results/clientpositive/bucketmapjoin11.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin11.q.out @@ -308,14 +308,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} @@ -331,30 +335,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 2354 Data size: 9418 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 2354 Data size: 9418 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 1177 Data size: 4709 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 1177 Data size: 4709 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -699,14 +707,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int), part (type: string) - 1 key (type: int), part (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int), part (type: string) + 1 key (type: int), part (type: string) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} @@ -722,30 +734,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int), part (type: string) - 1 key (type: int), part (type: string) - Position of Big Table: 0 - Statistics: Num rows: 2354 Data size: 9418 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 2354 Data size: 9418 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int), part (type: string) + 1 key (type: int), part (type: string) + Position of Big Table: 0 + Statistics: Num rows: 1177 Data size: 4709 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 1177 Data size: 4709 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin12.q.out ql/src/test/results/clientpositive/bucketmapjoin12.q.out index d97a89a..9253f4a 100644 --- ql/src/test/results/clientpositive/bucketmapjoin12.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin12.q.out @@ -220,14 +220,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} @@ -243,30 +247,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -507,14 +515,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -523,29 +535,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin13.q.out ql/src/test/results/clientpositive/bucketmapjoin13.q.out index c36aade..b380fab 100644 --- ql/src/test/results/clientpositive/bucketmapjoin13.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin13.q.out @@ -181,14 +181,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -197,29 +201,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -504,14 +512,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} @@ -527,30 +539,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -786,14 +802,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} @@ -809,30 +829,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -1070,14 +1094,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} @@ -1093,30 +1121,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin2.q.out ql/src/test/results/clientpositive/bucketmapjoin2.q.out index bcb581a..297412f 100644 --- ql/src/test/results/clientpositive/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin2.q.out @@ -222,14 +222,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} @@ -245,49 +249,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -766,14 +774,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} @@ -789,54 +801,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1407,14 +1423,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} @@ -1430,54 +1450,58 @@ STAGE PLANS: alias: a Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin3.q.out ql/src/test/results/clientpositive/bucketmapjoin3.q.out index 1b39788..7f307a0 100644 --- ql/src/test/results/clientpositive/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin3.q.out @@ -252,14 +252,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} @@ -275,49 +279,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -803,14 +811,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} @@ -826,54 +838,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin4.q.out ql/src/test/results/clientpositive/bucketmapjoin4.q.out index 8d3b0c6..f0f9aee 100644 --- ql/src/test/results/clientpositive/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin4.q.out @@ -192,14 +192,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} @@ -215,49 +219,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 0 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -670,14 +678,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} @@ -693,54 +705,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 14 Data size: 1512 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin5.q.out ql/src/test/results/clientpositive/bucketmapjoin5.q.out index 49f096a..79e1c3d 100644 --- ql/src/test/results/clientpositive/bucketmapjoin5.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin5.q.out @@ -242,14 +242,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} @@ -265,49 +269,53 @@ STAGE PLANS: alias: b Statistics: Num rows: 110 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 121 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 121 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 121 Data size: 12786 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -781,14 +789,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} @@ -804,54 +816,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 928 - rawDataSize 17038 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17966 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 928 + rawDataSize 17038 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17966 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin7.q.out ql/src/test/results/clientpositive/bucketmapjoin7.q.out index 8f4b826..76baf50 100644 --- ql/src/test/results/clientpositive/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin7.q.out @@ -189,14 +189,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {ds=2008-04-08/hr=0/srcbucket20.txt=[ds=2008-04-08/hr=0/srcbucket20.txt], ds=2008-04-08/hr=0/srcbucket21.txt=[ds=2008-04-08/hr=0/srcbucket21.txt]} @@ -212,47 +216,51 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col7 - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col7 + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin8.q.out ql/src/test/results/clientpositive/bucketmapjoin8.q.out index 94eebc9..94fdbde 100644 --- ql/src/test/results/clientpositive/bucketmapjoin8.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin8.q.out @@ -187,14 +187,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} @@ -210,30 +214,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -483,14 +491,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} @@ -506,30 +518,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin9.q.out ql/src/test/results/clientpositive/bucketmapjoin9.q.out index 6e1f205..c9f4c17 100644 --- ql/src/test/results/clientpositive/bucketmapjoin9.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin9.q.out @@ -195,14 +195,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 1050 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 525 Data size: 2100 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -211,29 +215,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 1155 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1155 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 577 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 577 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: @@ -516,14 +524,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -532,29 +544,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 755 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 378 Data size: 1514 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out index fc47322..751e32f 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out @@ -199,14 +199,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -215,48 +219,52 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 0 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 44 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 0 + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out index aa04241..3eb70d1 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out @@ -246,14 +246,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} @@ -269,49 +273,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6736 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 0 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 31 Data size: 3368 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out index 9b1d0a9..34abe4f 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out @@ -208,14 +208,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} @@ -231,44 +235,48 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -399,14 +407,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} @@ -422,44 +434,48 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -587,14 +603,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 (key + key) (type: double) - 1 UDFToDouble(key) (type: double) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 (key + key) (type: double) + 1 UDFToDouble(key) (type: double) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -603,43 +623,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 (key + key) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key + key) is not null (type: boolean) + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 (key + key) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 12 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 12 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 12 Data size: 2420 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -770,14 +794,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -786,43 +814,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -953,14 +985,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -969,43 +1005,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1136,14 +1176,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -1152,43 +1196,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1319,14 +1367,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -1335,43 +1387,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1502,14 +1558,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -1518,43 +1578,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1685,14 +1749,18 @@ STAGE PLANS: alias: r Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -1701,43 +1769,47 @@ STAGE PLANS: alias: l Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string), value (type: string) - 1 key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 1320 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out index 1d4e057..673cb49 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out @@ -103,26 +103,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -243,37 +246,41 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -311,37 +318,41 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -360,24 +371,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Extract File Output Operator @@ -504,26 +518,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -660,30 +677,33 @@ STAGE PLANS: TableScan alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -802,30 +822,33 @@ STAGE PLANS: TableScan alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), concat(value, value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), concat(value, value) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -956,43 +979,47 @@ STAGE PLANS: b:test_table2 TableScan alias: test_table2 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: test_table1 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1030,43 +1057,47 @@ STAGE PLANS: a:test_table1 TableScan alias: test_table1 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: test_table2 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1085,28 +1116,31 @@ STAGE PLANS: TableScan alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Extract File Output Operator diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out index e8024fb..8b24e73 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out @@ -79,26 +79,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -237,37 +240,41 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -305,37 +312,41 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -354,24 +365,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Extract File Output Operator diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out index 994fc3b..22b9a88 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out @@ -91,37 +91,41 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -159,37 +163,41 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -208,24 +216,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Extract File Output Operator @@ -344,43 +355,47 @@ STAGE PLANS: b:test_table2 TableScan alias: test_table2 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: test_table1 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -418,43 +433,47 @@ STAGE PLANS: a:test_table1 TableScan alias: test_table1 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: test_table2 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -473,28 +492,31 @@ STAGE PLANS: TableScan alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {_col1} - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col3) (type: string) - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: - - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col3) (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: - + Map-reduce partition columns: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Extract File Output Operator diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out index e794e4e..a839565 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out @@ -81,26 +81,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -216,26 +219,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -363,37 +369,41 @@ STAGE PLANS: b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -431,37 +441,41 @@ STAGE PLANS: a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -480,24 +494,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Reduce Operator Tree: Extract File Output Operator @@ -554,37 +571,41 @@ STAGE PLANS: subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -622,37 +643,41 @@ STAGE PLANS: subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -671,24 +696,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: +- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: +- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Reduce Operator Tree: Extract File Output Operator @@ -739,26 +767,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -892,26 +923,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -1066,37 +1100,41 @@ STAGE PLANS: subq2:subq1:b TableScan alias: b - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: -- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: -- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1134,37 +1172,41 @@ STAGE PLANS: subq2:subq1:a TableScan alias: a - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: b - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: -- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: -- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1183,24 +1225,27 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 91 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col8 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: -- - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Filter Operator + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col8 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col8) (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: -- + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Reduce Operator Tree: Extract File Output Operator diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_7.q.out index e5e047d..1827db1 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_7.q.out @@ -82,8 +82,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 0) or (key = 5)) (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and ((key = 0) or (key = 5))) (type: boolean) + Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -215,12 +215,12 @@ STAGE PLANS: alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 0) or (key = 5)) (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + predicate: (((key = 0) or (key = 5)) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -358,7 +358,7 @@ STAGE PLANS: alias: test_table1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and ((key = 0) or (key = 5))) (type: boolean) + predicate: (((key < 8) and key is not null) and ((key = 0) or (key = 5))) (type: boolean) Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out index 8f05d22..1044357 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out @@ -79,26 +79,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Select Operator - expressions: _col0 (type: int), _col5 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col0 (type: int), _col5 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -208,26 +211,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Select Operator - expressions: _col5 (type: int), _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Select Operator + expressions: _col5 (type: int), _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/cluster.q.out ql/src/test/results/clientpositive/cluster.q.out index 6f9b7e3..a163dd0 100644 --- ql/src/test/results/clientpositive/cluster.q.out +++ ql/src/test/results/clientpositive/cluster.q.out @@ -481,24 +481,24 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -508,11 +508,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -528,16 +528,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -577,25 +577,25 @@ STAGE PLANS: alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -605,11 +605,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -625,16 +625,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -674,25 +674,25 @@ STAGE PLANS: alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -702,11 +702,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -722,16 +722,16 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -771,24 +771,24 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -798,11 +798,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -818,16 +818,16 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/column_access_stats.q.out ql/src/test/results/clientpositive/column_access_stats.q.out index b180a9f..b7141bb 100644 --- ql/src/test/results/clientpositive/column_access_stats.q.out +++ ql/src/test/results/clientpositive/column_access_stats.q.out @@ -408,19 +408,25 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -429,14 +435,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -518,19 +524,19 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (val = 3) (type: boolean) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + predicate: ((val = 3) and key is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (val = 3) (type: boolean) + predicate: ((val = 3) and key is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -546,14 +552,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -602,22 +608,22 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 6) (type: boolean) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + predicate: ((key = 6) and val is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: val (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: ((key = 5) and val is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: val (type: string) @@ -636,14 +642,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -706,27 +712,33 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -735,11 +747,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -753,18 +765,21 @@ STAGE PLANS: TableScan alias: t3 Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -773,14 +788,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/correlationoptimizer1.q.out ql/src/test/results/clientpositive/correlationoptimizer1.q.out index eba3745..e41f8e3 100644 --- ql/src/test/results/clientpositive/correlationoptimizer1.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer1.q.out @@ -33,19 +33,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -54,17 +60,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -80,7 +86,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -88,11 +94,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash @@ -178,22 +184,28 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 60 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 3014 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -317,13 +329,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -331,33 +346,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -366,11 +384,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash @@ -463,28 +481,34 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -493,17 +517,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -519,7 +543,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -527,11 +551,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash @@ -617,31 +641,37 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 60 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 3014 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -2462,20 +2492,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2484,17 +2520,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2510,7 +2546,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2518,11 +2554,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)) mode: hash @@ -2609,20 +2645,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2631,17 +2673,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2657,7 +2699,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2665,11 +2707,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)) mode: hash @@ -2758,19 +2800,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2779,17 +2827,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2805,7 +2853,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2813,11 +2861,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash @@ -2904,19 +2952,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2925,17 +2979,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2951,7 +3005,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2959,11 +3013,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash diff --git ql/src/test/results/clientpositive/correlationoptimizer10.q.out ql/src/test/results/clientpositive/correlationoptimizer10.q.out index 635612e..27b082d 100644 --- ql/src/test/results/clientpositive/correlationoptimizer10.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer10.q.out @@ -45,19 +45,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -66,17 +72,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -92,7 +98,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -100,11 +106,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -118,26 +124,29 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -147,14 +156,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -225,39 +234,48 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -283,7 +301,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -304,7 +322,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -414,24 +432,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 20)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 20)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 20) and (key < 200)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key > 20)) and (key < 200)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -440,16 +458,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -465,16 +483,19 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -483,14 +504,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 727 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 727 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 727 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -568,38 +589,41 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 20)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 20)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 20) and (key < 200)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key > 20)) and (key < 200)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 13 Data size: 1418 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 816 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 13 Data size: 1418 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 816 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -632,7 +656,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 13 Data size: 1418 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 816 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -732,24 +756,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 180)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 180)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -758,16 +782,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -783,16 +807,19 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -801,14 +828,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -898,38 +925,41 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 180)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and (key > 180)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 41 Data size: 7014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 3606 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 41 Data size: 7014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 3606 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -962,7 +992,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 41 Data size: 7014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 3606 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer11.q.out ql/src/test/results/clientpositive/correlationoptimizer11.q.out index 1bf3914..22dc4d5 100644 --- ql/src/test/results/clientpositive/correlationoptimizer11.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer11.q.out @@ -69,19 +69,25 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 100 Data size: 1070 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 100 Data size: 1070 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 50 Data size: 535 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 50 Data size: 535 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -90,17 +96,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 110 Data size: 1177 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 110 Data size: 1177 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 110 Data size: 1177 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -116,7 +122,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 110 Data size: 1177 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -124,14 +130,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 288 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 288 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 55 Data size: 588 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 288 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -191,22 +197,28 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 100 Data size: 1070 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 100 Data size: 1070 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 50 Data size: 535 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 50 Data size: 535 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 125 Data size: 1261 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 634 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -302,19 +314,25 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -323,17 +341,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -349,7 +367,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -357,14 +375,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 101 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 54 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 101 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 54 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 101 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 54 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -435,22 +453,28 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 198 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer12.q.out ql/src/test/results/clientpositive/correlationoptimizer12.q.out index 86e7579..a04521c 100644 --- ql/src/test/results/clientpositive/correlationoptimizer12.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer12.q.out @@ -38,16 +38,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _wcol0 (type: bigint) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _wcol0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -64,7 +67,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -74,14 +77,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -104,16 +107,19 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _wcol0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _wcol0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/correlationoptimizer13.q.out ql/src/test/results/clientpositive/correlationoptimizer13.q.out index b0c7ebe..e3a8a5c 100644 --- ql/src/test/results/clientpositive/correlationoptimizer13.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer13.q.out @@ -60,23 +60,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c2 > 100) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c2 > 100) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c3 (type: string), c1 (type: int) outputColumnNames: c3, c1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c3 (type: string), c1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -84,11 +84,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -104,14 +104,14 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -121,11 +121,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: string), _col2 (type: bigint), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -140,15 +140,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: bigint) sort order: ++++++ - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: bigint), KEY.reducesinkkey5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -161,23 +161,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c1 < 120) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c3 (type: string), c1 (type: int) outputColumnNames: c3, c1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c3 (type: string), c1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -185,11 +185,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/correlationoptimizer14.q.out ql/src/test/results/clientpositive/correlationoptimizer14.q.out index 585fb89..cdbe61f 100644 --- ql/src/test/results/clientpositive/correlationoptimizer14.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer14.q.out @@ -45,15 +45,18 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) @@ -81,7 +84,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -91,14 +94,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -110,20 +113,23 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -168,15 +174,18 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -204,7 +213,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -214,14 +223,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -233,20 +242,23 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -291,16 +303,19 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -328,7 +343,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -338,14 +353,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -357,21 +372,24 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -469,38 +487,44 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 60 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 6444 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -523,9 +547,9 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 60 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 6444 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -642,16 +666,19 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: - - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: - + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -679,7 +706,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -689,14 +716,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -708,21 +735,24 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: - - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: - + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -769,15 +799,18 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -805,7 +838,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -815,14 +848,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -834,20 +867,23 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -892,16 +928,19 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -929,7 +968,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -939,14 +978,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -958,21 +997,24 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1070,38 +1112,44 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 60 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 6444 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1124,9 +1172,9 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 60 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 6444 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1243,33 +1291,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -1285,14 +1336,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -1302,14 +1353,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1321,21 +1372,24 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1433,44 +1487,50 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + TableScan + alias: x + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan - alias: x - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 31 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3114 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3114 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 46 Data size: 8944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 4671 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1495,13 +1555,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 2916 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1557 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 2916 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1557 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 46 Data size: 8944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 4671 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer15.q.out ql/src/test/results/clientpositive/correlationoptimizer15.q.out index eda3c0c..61b09f4 100644 --- ql/src/test/results/clientpositive/correlationoptimizer15.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer15.q.out @@ -36,19 +36,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -57,17 +63,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -83,7 +89,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -91,11 +97,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -109,17 +115,20 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -129,11 +138,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -148,15 +157,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string) sort order: +++ - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -248,30 +257,39 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -297,7 +315,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -318,7 +336,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer2.q.out ql/src/test/results/clientpositive/correlationoptimizer2.q.out index b1b64c1..553d86d 100644 --- ql/src/test/results/clientpositive/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer2.q.out @@ -38,22 +38,25 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -87,7 +90,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -97,11 +100,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)) mode: hash @@ -146,33 +149,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -231,56 +237,62 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1611 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1611 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -309,13 +321,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1611 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1611 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 30 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3222 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2040,22 +2052,25 @@ STAGE PLANS: TableScan alias: z Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2089,7 +2104,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -2099,11 +2114,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)) mode: hash @@ -2148,20 +2163,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2170,11 +2191,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2233,42 +2254,51 @@ STAGE PLANS: TableScan alias: z Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 60 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3338 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2282,7 +2312,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 30 Data size: 3122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1615 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2311,13 +2341,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1615 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1615 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 30 Data size: 3122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1615 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer3.q.out ql/src/test/results/clientpositive/correlationoptimizer3.q.out index f28d661..e4ffb79 100644 --- ql/src/test/results/clientpositive/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer3.q.out @@ -41,19 +41,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -62,17 +68,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -88,7 +94,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -96,11 +102,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -116,14 +122,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -133,11 +139,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)) mode: hash @@ -182,20 +188,26 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -204,11 +216,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -267,39 +279,51 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 119 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 6136 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -475,24 +499,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) tmp:d:x TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -500,74 +530,80 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + TableScan + alias: y + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan - alias: y - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 189 Data size: 19179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 9588 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -592,7 +628,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Mux Operator - Statistics: Num rows: 189 Data size: 19179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 9588 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -697,20 +733,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -719,11 +761,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -739,14 +781,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -756,11 +798,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)) mode: hash @@ -805,19 +847,25 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -826,17 +874,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -852,7 +900,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -860,11 +908,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -923,39 +971,51 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 119 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 6136 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1131,24 +1191,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) tmp:d:x TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -1156,64 +1222,70 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - TableScan - alias: y - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + alias: y + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 189 Data size: 19179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 9588 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1242,13 +1314,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 189 Data size: 19179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 9588 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer4.q.out ql/src/test/results/clientpositive/correlationoptimizer4.q.out index d309292..f9c2225 100644 --- ql/src/test/results/clientpositive/correlationoptimizer4.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer4.q.out @@ -76,27 +76,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: z Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -107,17 +116,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} 2 outputColumnNames: _col4 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int) outputColumnNames: _col4 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col4 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -133,7 +142,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -141,11 +150,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash @@ -233,30 +242,39 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: z Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 21 Data size: 90 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 51 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -381,28 +399,34 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) tmp:y TableScan alias: y Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) Stage: Stage-2 Map Reduce @@ -410,36 +434,39 @@ STAGE PLANS: TableScan alias: z Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 - 1 {key} - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col4 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col4 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 + 1 {key} + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) outputColumnNames: _col4 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col4 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col4 (type: int) + outputColumnNames: _col4 + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col4 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 8 Data size: 37 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -448,11 +475,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)) mode: hash diff --git ql/src/test/results/clientpositive/correlationoptimizer5.q.out ql/src/test/results/clientpositive/correlationoptimizer5.q.out index 4c77a14..df41d72 100644 --- ql/src/test/results/clientpositive/correlationoptimizer5.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer5.q.out @@ -115,20 +115,26 @@ STAGE PLANS: TableScan alias: m Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: n Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -137,11 +143,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -157,14 +163,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -173,14 +179,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -206,19 +212,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 724 Data size: 2897 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 724 Data size: 2897 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -227,11 +239,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -297,39 +309,51 @@ STAGE PLANS: TableScan alias: m Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE TableScan alias: n Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) TableScan alias: y Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 724 Data size: 2897 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 724 Data size: 2897 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 2956 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1479 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -486,13 +510,16 @@ STAGE PLANS: TableScan alias: m Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {val} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {val} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-10 Map Reduce @@ -500,27 +527,30 @@ STAGE PLANS: TableScan alias: n Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {val} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {val} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -636,14 +666,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -652,14 +682,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 878 Data size: 3517 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -677,13 +707,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 724 Data size: 2897 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-11 Map Reduce @@ -691,27 +724,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/correlationoptimizer6.q.out ql/src/test/results/clientpositive/correlationoptimizer6.q.out index 1e6e6ae..1a7441e 100644 --- ql/src/test/results/clientpositive/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer6.q.out @@ -39,19 +39,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -60,17 +66,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -86,7 +92,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -94,11 +100,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -114,14 +120,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -131,14 +137,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -150,19 +156,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -171,17 +183,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -197,7 +209,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -205,11 +217,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -285,38 +297,50 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 120 Data size: 12056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 6028 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -487,24 +511,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) yy:x TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -512,80 +542,86 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 65 Data size: 6630 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3314 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 64 Data size: 6528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3314 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -610,13 +646,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 64 Data size: 6528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3314 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -710,33 +746,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -752,16 +791,19 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -770,14 +812,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col4, _col5 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -847,35 +889,41 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 60 Data size: 6028 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 3014 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 90 Data size: 9042 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 4521 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -900,13 +948,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1507 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1507 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 90 Data size: 9042 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 4521 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -999,19 +1047,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1020,17 +1074,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1046,7 +1100,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1054,11 +1108,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1074,16 +1128,19 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1092,14 +1149,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col4, _col5 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1169,32 +1226,41 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 118 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 59 Data size: 5920 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 118 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 59 Data size: 5920 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1239,7 +1305,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 118 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 59 Data size: 5920 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1332,19 +1398,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1353,17 +1425,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1379,7 +1451,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1387,11 +1459,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -1405,17 +1477,20 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -1425,14 +1500,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1524,30 +1599,39 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1573,7 +1657,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1594,7 +1678,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 62 Data size: 6244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3122 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1711,19 +1795,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1732,17 +1822,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1758,7 +1848,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1766,11 +1856,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1784,26 +1874,32 @@ STAGE PLANS: TableScan alias: zz Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1814,14 +1910,14 @@ STAGE PLANS: 1 2 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col8, _col9 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col8 (type: string), _col9 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1923,40 +2019,52 @@ STAGE PLANS: TableScan alias: zz Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 88 Data size: 8826 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 352 Data size: 35304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2003,7 +2111,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 352 Data size: 35304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2124,19 +2232,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2145,17 +2259,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2171,7 +2285,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2179,11 +2293,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2197,26 +2311,32 @@ STAGE PLANS: TableScan alias: zz Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2227,14 +2347,14 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 outputColumnNames: _col0, _col4, _col5 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2330,40 +2450,52 @@ STAGE PLANS: TableScan alias: zz Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 88 Data size: 8826 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 352 Data size: 35304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2410,7 +2542,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 352 Data size: 35304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 17652 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2535,33 +2667,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -2577,14 +2712,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -2594,17 +2729,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col1), sum(_col3) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2620,7 +2755,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2628,11 +2763,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2648,17 +2783,20 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) TableScan alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -2667,14 +2805,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2686,33 +2824,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2814,65 +2955,74 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 89 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 6020 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 44 Data size: 5853 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2943 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 44 Data size: 5853 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2943 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 88 Data size: 11706 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 44 Data size: 5886 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2898,7 +3048,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 89 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 6020 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2923,13 +3073,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 44 Data size: 5853 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2943 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 44 Data size: 5853 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 2943 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 88 Data size: 11706 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 44 Data size: 5886 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2955,7 +3105,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 89 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 6020 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2976,7 +3126,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 89 Data size: 11840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 6020 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -3102,20 +3252,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -3124,17 +3280,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3150,7 +3306,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -3158,11 +3314,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3178,14 +3334,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -3195,11 +3351,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3214,15 +3370,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) sort order: +++++ - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -3234,19 +3390,25 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -3255,17 +3417,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3281,7 +3443,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -3289,11 +3451,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -3371,20 +3533,26 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -3393,17 +3561,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3419,7 +3587,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -3427,11 +3595,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3447,27 +3615,33 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: bigint) TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 35 Data size: 3577 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -3493,7 +3667,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 35 Data size: 3577 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -3514,7 +3688,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 35 Data size: 3577 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -3611,13 +3785,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -3625,33 +3802,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -3660,11 +3840,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -3683,13 +3863,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -3699,55 +3882,58 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: bigint) TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 237 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 118 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 33 Data size: 3382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1664 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 49 Data size: 5021 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 2496 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -3768,7 +3954,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 49 Data size: 5021 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 2496 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer7.q.out ql/src/test/results/clientpositive/correlationoptimizer7.q.out index ea8abc8..269a6cf 100644 --- ql/src/test/results/clientpositive/correlationoptimizer7.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer7.q.out @@ -37,13 +37,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -51,33 +54,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -86,11 +92,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -109,13 +115,16 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -131,14 +140,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -220,13 +229,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -234,59 +246,65 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 64 Data size: 6609 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3412 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1706 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1706 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 96 Data size: 9913 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 5118 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -307,7 +325,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 96 Data size: 9913 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 5118 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -416,13 +434,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -430,33 +451,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -465,11 +489,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -488,13 +512,16 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -510,14 +537,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -599,13 +626,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -613,59 +643,65 @@ STAGE PLANS: TableScan alias: yy Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 64 Data size: 6609 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3412 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1706 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1706 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 96 Data size: 9913 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 5118 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -686,7 +722,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 96 Data size: 9913 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 5118 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/correlationoptimizer8.q.out ql/src/test/results/clientpositive/correlationoptimizer8.q.out index 6973111..662038d 100644 --- ql/src/test/results/clientpositive/correlationoptimizer8.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer8.q.out @@ -46,23 +46,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 20) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -70,11 +70,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -87,31 +87,34 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Union - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -120,14 +123,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -140,23 +143,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -164,11 +167,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -247,72 +250,75 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 20) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 39 Data size: 4022 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 2220 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1959 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1959 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 38 Data size: 3918 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2114 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 77 Data size: 7940 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 4334 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -337,15 +343,15 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1959 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1959 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 38 Data size: 3918 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2114 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 77 Data size: 7940 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 4334 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -366,7 +372,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 77 Data size: 7940 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 4334 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -932,23 +938,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 20) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -956,11 +962,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -973,31 +979,34 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 13 Data size: 1702 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1702 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 901 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Union - Statistics: Num rows: 13 Data size: 1702 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1702 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 901 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1006,14 +1015,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 14 Data size: 1872 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 991 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 14 Data size: 1872 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 991 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 14 Data size: 1872 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 991 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1026,23 +1035,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1050,11 +1059,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/correlationoptimizer9.q.out ql/src/test/results/clientpositive/correlationoptimizer9.q.out index 5ae58e9..6be395b 100644 --- ql/src/test/results/clientpositive/correlationoptimizer9.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer9.q.out @@ -53,23 +53,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c2 > 100) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: ((c2 > 100) and c2 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c2 (type: int) outputColumnNames: c2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c2 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -77,11 +77,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -97,14 +97,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -114,14 +114,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col1 (type: bigint), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -134,23 +134,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c1 < 120) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: ((c1 < 120) and c1 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int) outputColumnNames: c1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -158,11 +158,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -233,61 +233,61 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c2 > 100) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: ((c2 > 100) and c2 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c2 (type: int) outputColumnNames: c2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c2 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c1 < 120) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: ((c1 < 120) and c1 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int) outputColumnNames: c1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -312,13 +312,13 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -402,23 +402,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c2 > 100) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c2 > 100) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -426,11 +426,11 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -446,14 +446,14 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -463,14 +463,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: string), _col2 (type: bigint), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -483,23 +483,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c1 < 120) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -507,11 +507,11 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -580,61 +580,61 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c2 > 100) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c2 > 100) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) TableScan alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (c1 < 120) (type: boolean) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -659,13 +659,13 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/cross_join.q.out ql/src/test/results/clientpositive/cross_join.q.out index ef48a38..94bcb72 100644 --- ql/src/test/results/clientpositive/cross_join.q.out +++ ql/src/test/results/clientpositive/cross_join.q.out @@ -125,19 +125,25 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -146,14 +152,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/cross_product_check_1.q.out ql/src/test/results/clientpositive/cross_product_check_1.q.out index af380fc..fec2f7b 100644 --- ql/src/test/results/clientpositive/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/cross_product_check_1.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [d1, d2, a]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [d1, d2, a]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -89,21 +89,27 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -112,7 +118,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -126,7 +132,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: a @@ -162,7 +168,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [a, od1]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[16][tables = [a, od1]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -186,19 +192,25 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -207,19 +219,19 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -235,17 +247,17 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -259,7 +271,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan alias: a @@ -420,7 +432,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [ss, od1]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[21][tables = [ss, od1]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -443,19 +455,25 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -464,19 +482,19 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -492,17 +510,17 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -516,7 +534,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan Reduce Output Operator diff --git ql/src/test/results/clientpositive/cross_product_check_2.q.out ql/src/test/results/clientpositive/cross_product_check_2.q.out index bff4ed9..74abda7 100644 --- ql/src/test/results/clientpositive/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/cross_product_check_2.q.out @@ -85,7 +85,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -121,13 +121,16 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -135,39 +138,42 @@ STAGE PLANS: TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col4} {_col5} + 0 {key} {value} 1 {key} {value} keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col4} {_col5} + 1 {key} {value} + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -177,7 +183,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -207,13 +213,16 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -221,34 +230,37 @@ STAGE PLANS: TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -256,11 +268,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -462,9 +474,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[88][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[58][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [ss, od1]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[100][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[66][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[21][tables = [ss, od1]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -618,7 +630,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan Reduce Output Operator @@ -657,13 +669,16 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -671,34 +686,37 @@ STAGE PLANS: TableScan alias: d2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -706,11 +724,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/explain_logical.q.out ql/src/test/results/clientpositive/explain_logical.q.out index 0047133..bb26e8c 100644 --- ql/src/test/results/clientpositive/explain_logical.q.out +++ ql/src/test/results/clientpositive/explain_logical.q.out @@ -361,48 +361,54 @@ s1 TableScan (TS_1) alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_2) - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Join Operator (JOIN_4) - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col0} - outputColumnNames: _col0, _col5 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_6) - expressions: _col0 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - File Output Operator (FS_7) - compressed: false - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator (FIL_11) + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_3) + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Join Operator (JOIN_6) + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col5 + Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_8) + expressions: _col0 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + File Output Operator (FS_9) + compressed: false + Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe s2 TableScan (TS_0) alias: s2 Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_3) - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Join Operator (JOIN_4) - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col0} - outputColumnNames: _col0, _col5 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Filter Operator (FIL_10) + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_5) + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Join Operator (JOIN_6) + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col5 + Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: -- With views EXPLAIN LOGICAL SELECT * FROM V1 @@ -505,52 +511,58 @@ v3:src1:srcpart TableScan (TS_0) alias: srcpart Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_2) - expressions: key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_4) - key expressions: _col1 (type: string) + Filter Operator (FIL_14) + predicate: key is not null (type: boolean) + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_2) + expressions: key (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_5) + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Join Operator (JOIN_8) + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col1, _col4 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_10) + expressions: _col1 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + File Output Operator (FS_12) + compressed: false + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +v3:src2 + TableScan (TS_3) + alias: src2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator (FIL_13) + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_7) + key expressions: key (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Join Operator (JOIN_6) + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Join Operator (JOIN_8) condition map: Inner Join 0 to 1 condition expressions: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col1, _col4 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_8) - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - File Output Operator (FS_10) - compressed: false - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -v3:src2 - TableScan (TS_3) - alias: src2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_5) - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Join Operator (JOIN_6) - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col0} - outputColumnNames: _col1, _col4 - Statistics: Num rows: 255 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: EXPLAIN LOGICAL SELECT * FROM V4 PREHOOK: type: QUERY @@ -577,51 +589,80 @@ v4:src1:src TableScan (TS_3) alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_4) - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_6) - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Join Operator (JOIN_9) - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col1} - 2 {VALUE._col0} - outputColumnNames: _col0, _col4, _col6 - Statistics: Num rows: 255 Data size: 51145 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_10) - expressions: _col0 (type: string), _col4 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 255 Data size: 51145 Basic stats: COMPLETE Column stats: NONE - File Output Operator (FS_12) - compressed: false - Statistics: Num rows: 255 Data size: 51145 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator (FIL_16) + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_4) + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_7) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Join Operator (JOIN_12) + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col1} + 2 {VALUE._col0} + outputColumnNames: _col0, _col4, _col6 + Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_13) + expressions: _col0 (type: string), _col4 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + File Output Operator (FS_15) + compressed: false + Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe v4:src2:srcpart TableScan (TS_0) alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_2) - expressions: key (type: string), value (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_7) - key expressions: _col1 (type: string) + Filter Operator (FIL_18) + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_2) + expressions: key (type: string), value (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_9) + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Join Operator (JOIN_12) + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col1} + 2 {VALUE._col0} + outputColumnNames: _col0, _col4, _col6 + Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE +v4:src3 + TableScan (TS_5) + alias: src3 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator (FIL_17) + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_11) + key expressions: key (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Join Operator (JOIN_9) + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Join Operator (JOIN_12) condition map: Inner Join 0 to 1 Inner Join 1 to 2 @@ -630,27 +671,7 @@ v4:src2:srcpart 1 {VALUE._col1} 2 {VALUE._col0} outputColumnNames: _col0, _col4, _col6 - Statistics: Num rows: 255 Data size: 51145 Basic stats: COMPLETE Column stats: NONE -v4:src3 - TableScan (TS_5) - alias: src3 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_8) - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Join Operator (JOIN_9) - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col1} - 2 {VALUE._col0} - outputColumnNames: _col0, _col4, _col6 - Statistics: Num rows: 255 Data size: 51145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: -- The table should show up in the explain logical even if none -- of the partitions are selected. @@ -782,82 +803,88 @@ s1:src TableScan (TS_0) alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_1) - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator (GBY_2) - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_3) - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Group By Operator (GBY_4) - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_5) - expressions: _col0 (type: string), _col1 (type: bigint) + Filter Operator (FIL_17) + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_1) + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator (GBY_2) + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_3) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Group By Operator (GBY_4) + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_7) - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Join Operator (JOIN_9) - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {VALUE._col0} - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator (SEL_10) - expressions: _col0 (type: string), _col1 (type: bigint), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_11) - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: string) - Select Operator (SEL_12) - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator (FS_13) - compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_5) + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_8) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Join Operator (JOIN_11) + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator (SEL_12) + expressions: _col0 (type: string), _col1 (type: bigint), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_13) + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: string) + Select Operator (SEL_14) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator (FS_15) + compressed: false + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe s2 TableScan (TS_6) alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator (RS_8) - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Join Operator (JOIN_9) - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {VALUE._col0} - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator (FIL_16) + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator (RS_10) + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Join Operator (JOIN_11) + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/explain_rearrange.q.out ql/src/test/results/clientpositive/explain_rearrange.q.out index 0451fb3..ad9e991 100644 --- ql/src/test/results/clientpositive/explain_rearrange.q.out +++ ql/src/test/results/clientpositive/explain_rearrange.q.out @@ -75,29 +75,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -260,29 +263,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -364,29 +370,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -549,29 +558,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -653,29 +665,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -698,29 +713,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -942,29 +960,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -987,29 +1008,32 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/filter_join_breaktask.q.out ql/src/test/results/clientpositive/filter_join_breaktask.q.out index 1bc561a..b379f86 100644 --- ql/src/test/results/clientpositive/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/filter_join_breaktask.q.out @@ -263,13 +263,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (value <> '') (type: boolean) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE tag: 1 Path -> Alias: #### A masked pattern was here #### @@ -351,17 +351,17 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col11 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col11 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out index afdefb2..6f9f10b 100644 --- ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out +++ ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out @@ -51,7 +51,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (a < 3) (type: boolean) + predicate: ((a < 3) and a is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -132,7 +132,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (a < 3) (type: boolean) + predicate: ((a < 3) and a is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -237,7 +237,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (a < 3) (type: boolean) + predicate: ((a < 3) and a is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -342,7 +342,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (a < 3) (type: boolean) + predicate: ((a < 3) and a is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) diff --git ql/src/test/results/clientpositive/groupby_position.q.out ql/src/test/results/clientpositive/groupby_position.q.out index 38b7d46..07e9641 100644 --- ql/src/test/results/clientpositive/groupby_position.q.out +++ ql/src/test/results/clientpositive/groupby_position.q.out @@ -575,23 +575,23 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 10) and (key < 20)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > 10) and (key < 20)) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(DISTINCT substr(value, 5)) keys: key (type: string), value (type: string), substr(value, 5) (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col2:0._col0) @@ -675,22 +675,22 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 15) and (key < 25)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > 15) and (key < 25)) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) diff --git ql/src/test/results/clientpositive/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index 32919ec..38a0678 100644 --- ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -3760,52 +3760,60 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col1 (type: bigint) TableScan alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3868,17 +3876,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), UDFToInteger((_col1 + _col3)) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -4070,23 +4078,27 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string), val (type: string) - outputColumnNames: key, val - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string), val (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col2 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: key, val + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string), val (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col2 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4147,11 +4159,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -4177,34 +4189,38 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col1 (type: bigint) TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: bigint) Path -> Alias: @@ -4290,17 +4306,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index cb3bdcf..ac54e7d 100644 --- ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -4080,52 +4080,60 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col1 (type: bigint) TableScan alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4188,17 +4196,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), UDFToInteger((_col1 + _col3)) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -4391,23 +4399,27 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string), val (type: string) - outputColumnNames: key, val - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string), val (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: rand() (type: double) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col2 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: key, val + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string), val (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: rand() (type: double) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col2 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4468,7 +4480,7 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: partials outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -4496,7 +4508,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) Path -> Alias: @@ -4531,11 +4543,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: final outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -4561,34 +4573,38 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: final - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col1 (type: bigint) TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: bigint) Path -> Alias: @@ -4674,17 +4690,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/index_auto_mult_tables.q.out ql/src/test/results/clientpositive/index_auto_mult_tables.q.out index d8f588f..7603d26 100644 --- ql/src/test/results/clientpositive/index_auto_mult_tables.q.out +++ ql/src/test/results/clientpositive/index_auto_mult_tables.q.out @@ -22,18 +22,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -49,14 +49,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -243,22 +243,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + filterExpr: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: a - filterExpr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + filterExpr: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -274,14 +274,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out index eb25824..ecbd844 100644 --- ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out +++ ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out @@ -22,18 +22,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -49,14 +49,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -243,22 +243,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + filterExpr: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: a - filterExpr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + filterExpr: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -274,14 +274,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/index_auto_self_join.q.out ql/src/test/results/clientpositive/index_auto_self_join.q.out index e42bfd0..7bd782c 100644 --- ql/src/test/results/clientpositive/index_auto_self_join.q.out +++ ql/src/test/results/clientpositive/index_auto_self_join.q.out @@ -20,25 +20,25 @@ STAGE PLANS: alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 70) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string) TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 80) and (key < 100)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string) Reduce Operator Tree: Join Operator @@ -48,14 +48,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col4 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -165,29 +165,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: ((key > 70) and (key < 90)) (type: boolean) + filterExpr: ((value is not null and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 70) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string) TableScan alias: a - filterExpr: ((key > 80) and (key < 100)) (type: boolean) + filterExpr: ((value is not null and (key > 80)) and (key < 100)) (type: boolean) Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 80) and (key < 100)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string) Reduce Operator Tree: Join Operator @@ -197,14 +197,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col4 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/index_bitmap3.q.out ql/src/test/results/clientpositive/index_bitmap3.q.out index e0df3e0..7923902 100644 --- ql/src/test/results/clientpositive/index_bitmap3.q.out +++ ql/src/test/results/clientpositive/index_bitmap3.q.out @@ -1,7 +1,11 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD PREHOOK: type: CREATEINDEX -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD POSTHOOK: type: CREATEINDEX STAGE DEPENDENCIES: @@ -56,19 +60,19 @@ POSTHOOK: Lineage: default__src_src2_index__._bitmaps EXPRESSION [(src)src.Field POSTHOOK: Lineage: default__src_src2_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ] POSTHOOK: Lineage: default__src_src2_index__._offset SIMPLE [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ] POSTHOOK: Lineage: default__src_src2_index__.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM default__src_src1_index__ ORDER BY key +PREHOOK: query: SELECT * FROM default__src_src1_index__ PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src1_index__ #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM default__src_src1_index__ ORDER BY key +POSTHOOK: query: SELECT * FROM default__src_src1_index__ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ #### A masked pattern was here #### -PREHOOK: query: SELECT * FROM default__src_src2_index__ ORDER BY value +PREHOOK: query: SELECT * FROM default__src_src2_index__ PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM default__src_src2_index__ ORDER BY value +POSTHOOK: query: SELECT * FROM default__src_src2_index__ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### @@ -107,56 +111,56 @@ STAGE PLANS: alias: default__src_src1_index__ Statistics: Num rows: 500 Data size: 46311 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 0) (type: boolean) - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE + predicate: (((key = 0) and _bucketname is not null) and _offset is not null) (type: boolean) + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: array) + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: array) TableScan alias: default__src_src2_index__ Statistics: Num rows: 500 Data size: 48311 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value = 'val_0') (type: boolean) - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + predicate: (((value = 'val_0') and _bucketname is not null) and _offset is not null) (type: boolean) + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} - 1 {VALUE._col2} + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} + 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 275 Data size: 25470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 69 Data size: 6418 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (not EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(_col2,_col5))) (type: boolean) - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: collect_set(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -172,7 +176,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: array) Reduce Operator Tree: Group By Operator @@ -180,14 +184,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -226,22 +230,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ POSTHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### -PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 0 val_0 0 val_0 -PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/index_bitmap_auto.q.out ql/src/test/results/clientpositive/index_bitmap_auto.q.out index 013cf77..5ad785d 100644 --- ql/src/test/results/clientpositive/index_bitmap_auto.q.out +++ ql/src/test/results/clientpositive/index_bitmap_auto.q.out @@ -1,12 +1,16 @@ -PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- try the query without indexing, with manual indexing, and with automatic indexing -- without indexing -SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +SELECT key, value FROM src WHERE key=0 AND value = "val_0" PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- try the query without indexing, with manual indexing, and with automatic indexing -- without indexing -SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +SELECT key, value FROM src WHERE key=0 AND value = "val_0" POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -73,19 +77,19 @@ POSTHOOK: Lineage: default__src_src2_index__._bitmaps EXPRESSION [(src)src.Field POSTHOOK: Lineage: default__src_src2_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ] POSTHOOK: Lineage: default__src_src2_index__._offset SIMPLE [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ] POSTHOOK: Lineage: default__src_src2_index__.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM default__src_src1_index__ ORDER BY key +PREHOOK: query: SELECT * FROM default__src_src1_index__ PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src1_index__ #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM default__src_src1_index__ ORDER BY key +POSTHOOK: query: SELECT * FROM default__src_src1_index__ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ #### A masked pattern was here #### -PREHOOK: query: SELECT * FROM default__src_src2_index__ ORDER BY value +PREHOOK: query: SELECT * FROM default__src_src2_index__ PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM default__src_src2_index__ ORDER BY value +POSTHOOK: query: SELECT * FROM default__src_src2_index__ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### @@ -126,56 +130,56 @@ STAGE PLANS: alias: default__src_src1_index__ Statistics: Num rows: 500 Data size: 46311 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 0) (type: boolean) - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE + predicate: (((key = 0) and _bucketname is not null) and _offset is not null) (type: boolean) + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 250 Data size: 23155 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: array) + Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: array) TableScan alias: default__src_src2_index__ Statistics: Num rows: 500 Data size: 48311 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value = 'val_0') (type: boolean) - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + predicate: (((value = 'val_0') and _bucketname is not null) and _offset is not null) (type: boolean) + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 250 Data size: 24155 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} - 1 {VALUE._col2} + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} + 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 275 Data size: 25470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 69 Data size: 6418 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (not EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(_col2,_col5))) (type: boolean) - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: collect_set(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -191,7 +195,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 138 Data size: 12781 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: array) Reduce Operator Tree: Group By Operator @@ -199,14 +203,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 6390 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -245,24 +249,24 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ POSTHOOK: Input: default@default__src_src2_index__ #### A masked pattern was here #### -PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 0 val_0 0 val_0 0 val_0 -PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src1_index__ PREHOOK: Input: default@default__src_src2_index__ PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key +POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ POSTHOOK: Input: default@default__src_src2_index__ diff --git ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out index 57d02bf..a5691e6 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out @@ -246,28 +246,34 @@ STAGE PLANS: mode: final outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col1) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col1) (type: double) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) + Filter Operator + predicate: UDFToDouble(_col1) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1503 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 1503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col1) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col1) (type: double) + Statistics: Num rows: 15 Data size: 1503 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col1) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col1) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(value) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col1) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col1) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -276,14 +282,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -395,26 +401,29 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Select Operator - expressions: _col0 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table_out + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Select Operator + expressions: _col0 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table_out Stage: Stage-7 Conditional Operator @@ -547,29 +556,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col5 - Select Operator - expressions: _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col5 - Group By Operator - aggregations: count() - keys: _col5 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col5 (type: string) + outputColumnNames: _col5 + Group By Operator + aggregations: count() + keys: _col5 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/innerjoin.q.out ql/src/test/results/clientpositive/innerjoin.q.out index 34f457a..c8291e6 100644 --- ql/src/test/results/clientpositive/innerjoin.q.out +++ ql/src/test/results/clientpositive/innerjoin.q.out @@ -29,20 +29,26 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -51,14 +57,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/input39.q.out ql/src/test/results/clientpositive/input39.q.out index 3652e0b..3e48b8f 100644 --- ql/src/test/results/clientpositive/input39.q.out +++ ql/src/test/results/clientpositive/input39.q.out @@ -73,24 +73,24 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: ((((hash(rand(460476415)) & 2147483647) % 32) = 0) and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE TableScan alias: t1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: ((((hash(rand(460476415)) & 2147483647) % 32) = 0) and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -98,9 +98,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash diff --git ql/src/test/results/clientpositive/join1.q.out ql/src/test/results/clientpositive/join1.q.out index 6142c43..7fbf854 100644 --- ql/src/test/results/clientpositive/join1.q.out +++ ql/src/test/results/clientpositive/join1.q.out @@ -25,20 +25,26 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -47,14 +53,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join10.q.out ql/src/test/results/clientpositive/join10.q.out index c8a0d6a..e17b547 100644 --- ql/src/test/results/clientpositive/join10.q.out +++ ql/src/test/results/clientpositive/join10.q.out @@ -23,28 +23,34 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -53,14 +59,14 @@ STAGE PLANS: 0 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join11.q.out ql/src/test/results/clientpositive/join11.q.out index a6dfdaa..abdc863 100644 --- ql/src/test/results/clientpositive/join11.q.out +++ ql/src/test/results/clientpositive/join11.q.out @@ -26,33 +26,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -61,14 +61,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join12.q.out ql/src/test/results/clientpositive/join12.q.out index e351ba6..0a3d642 100644 --- ql/src/test/results/clientpositive/join12.q.out +++ ql/src/test/results/clientpositive/join12.q.out @@ -32,48 +32,48 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 80) and (key < 100)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -84,14 +84,14 @@ STAGE PLANS: 1 {VALUE._col0} 2 outputColumnNames: _col0, _col3 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join13.q.out ql/src/test/results/clientpositive/join13.q.out index 3adcfb4..432c1e6 100644 --- ql/src/test/results/clientpositive/join13.q.out +++ ql/src/test/results/clientpositive/join13.q.out @@ -33,33 +33,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -68,13 +68,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 + _col2) is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -84,23 +87,23 @@ STAGE PLANS: key expressions: (_col0 + _col2) (type: double) sort order: + Map-reduce partition columns: (_col0 + _col2) (type: double) - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col3 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 200) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -109,14 +112,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col3} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 22 Data size: 2302 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 2302 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 2302 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 18f829d..b37ae6b 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -30,25 +30,25 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -57,14 +57,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join15.q.out ql/src/test/results/clientpositive/join15.q.out index b72f3e8..2caf4d5 100644 --- ql/src/test/results/clientpositive/join15.q.out +++ ql/src/test/results/clientpositive/join15.q.out @@ -16,21 +16,27 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -39,11 +45,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -58,15 +64,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join16.q.out ql/src/test/results/clientpositive/join16.q.out index 124214a..725cd08 100644 --- ql/src/test/results/clientpositive/join16.q.out +++ ql/src/test/results/clientpositive/join16.q.out @@ -14,28 +14,28 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and (key > 20)) and (value < 200)) (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: tab Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 20) and (value < 200)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (value < 200)) (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -44,14 +44,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey1} outputColumnNames: _col0, _col3 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 440 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 440 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 440 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join17.q.out ql/src/test/results/clientpositive/join17.q.out index 9439f6d..76741bb 100644 --- ql/src/test/results/clientpositive/join17.q.out +++ ql/src/test/results/clientpositive/join17.q.out @@ -64,24 +64,32 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: value (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: value (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -140,17 +148,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col4) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/join19.q.out ql/src/test/results/clientpositive/join19.q.out index 803d7a4..25cb4b3 100644 --- ql/src/test/results/clientpositive/join19.q.out +++ ql/src/test/results/clientpositive/join19.q.out @@ -131,7 +131,7 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL') (type: boolean) + predicate: ((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL') and subject is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string), object (type: string) @@ -147,7 +147,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (predicate = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_from') (type: boolean) + predicate: (((predicate = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_from') and object is not null) and subject is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string), object (type: string) @@ -163,7 +163,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: ((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (object = 'http://ontos/OntosMiner/Common.English/ontology#Citation')) (type: boolean) + predicate: (((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (object = 'http://ontos/OntosMiner/Common.English/ontology#Citation')) and subject is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string) @@ -206,7 +206,7 @@ STAGE PLANS: alias: t5 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (predicate = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_to') (type: boolean) + predicate: (((predicate = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_to') and subject is not null) and object is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string), object (type: string) @@ -222,7 +222,7 @@ STAGE PLANS: alias: t4 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: ((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (object = 'http://ontos/OntosMiner/Common.English/ontology#Author')) (type: boolean) + predicate: (((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (object = 'http://ontos/OntosMiner/Common.English/ontology#Author')) and subject is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string) @@ -258,7 +258,7 @@ STAGE PLANS: alias: t6 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL') (type: boolean) + predicate: ((predicate = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL') and subject is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: subject (type: string), object (type: string) diff --git ql/src/test/results/clientpositive/join2.q.out ql/src/test/results/clientpositive/join2.q.out index ce87da7..c80f830 100644 --- ql/src/test/results/clientpositive/join2.q.out +++ ql/src/test/results/clientpositive/join2.q.out @@ -26,19 +26,25 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -47,13 +53,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 + _col4) is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1649 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -63,17 +72,20 @@ STAGE PLANS: key expressions: (_col0 + _col4) (type: double) sort order: + Map-reduce partition columns: (_col0 + _col4) (type: double) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1649 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan alias: src3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -82,14 +94,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col1} outputColumnNames: _col0, _col9 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1813 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join22.q.out ql/src/test/results/clientpositive/join22.q.out index 24685a3..f7f0e34 100644 --- ql/src/test/results/clientpositive/join22.q.out +++ ql/src/test/results/clientpositive/join22.q.out @@ -16,20 +16,26 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -38,11 +44,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -58,16 +64,19 @@ STAGE PLANS: key expressions: _col2 (type: string) sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: string) TableScan alias: src4 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -76,14 +85,14 @@ STAGE PLANS: 0 1 {VALUE._col2} outputColumnNames: _col7 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col7 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join25.q.out ql/src/test/results/clientpositive/join25.q.out index 6094da7..d8c7b7d 100644 --- ql/src/test/results/clientpositive/join25.q.out +++ ql/src/test/results/clientpositive/join25.q.out @@ -42,13 +42,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -56,29 +59,32 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join26.q.out ql/src/test/results/clientpositive/join26.q.out index eec9eaa..bf8cf57 100644 --- ql/src/test/results/clientpositive/join26.q.out +++ ql/src/test/results/clientpositive/join26.q.out @@ -127,31 +127,39 @@ STAGE PLANS: alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 y TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 Stage: Stage-1 Map Reduce @@ -160,51 +168,55 @@ STAGE PLANS: alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key} - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col5, _col9 - Position of Big Table: 2 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col5, _col9 + Position of Big Table: 2 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/join27.q.out ql/src/test/results/clientpositive/join27.q.out index a36a44c..351a2da 100644 --- ql/src/test/results/clientpositive/join27.q.out +++ ql/src/test/results/clientpositive/join27.q.out @@ -42,13 +42,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - keys: - 0 value (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) Stage: Stage-1 Map Reduce @@ -56,29 +59,32 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join28.q.out ql/src/test/results/clientpositive/join28.q.out index 59922da..1e4e09f 100644 --- ql/src/test/results/clientpositive/join28.q.out +++ ql/src/test/results/clientpositive/join28.q.out @@ -50,24 +50,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) z TableScan alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-6 Map Reduce @@ -75,44 +81,47 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join29.q.out ql/src/test/results/clientpositive/join29.q.out index 9a208cb..aafd07e 100644 --- ql/src/test/results/clientpositive/join29.q.out +++ ql/src/test/results/clientpositive/join29.q.out @@ -44,33 +44,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -190,14 +193,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -207,14 +210,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int), UDFToInteger(_col3) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -227,33 +230,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/join3.q.out ql/src/test/results/clientpositive/join3.q.out index a4963aa..e4c34e5 100644 --- ql/src/test/results/clientpositive/join3.q.out +++ ql/src/test/results/clientpositive/join3.q.out @@ -25,28 +25,37 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -57,14 +66,14 @@ STAGE PLANS: 1 2 {VALUE._col0} outputColumnNames: _col0, _col9 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col9 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join30.q.out ql/src/test/results/clientpositive/join30.q.out index 92c258a..e5370fe 100644 --- ql/src/test/results/clientpositive/join30.q.out +++ ql/src/test/results/clientpositive/join30.q.out @@ -35,13 +35,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -49,33 +52,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -84,14 +90,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join31.q.out ql/src/test/results/clientpositive/join31.q.out index 6ce6843..7d10a53 100644 --- ql/src/test/results/clientpositive/join31.q.out +++ ql/src/test/results/clientpositive/join31.q.out @@ -47,33 +47,36 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -140,7 +143,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -148,14 +151,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -231,13 +234,13 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -246,17 +249,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -270,33 +273,36 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/join32.q.out ql/src/test/results/clientpositive/join32.q.out index 56024b6..ff0d7cc 100644 --- ql/src/test/results/clientpositive/join32.q.out +++ ql/src/test/results/clientpositive/join32.q.out @@ -166,27 +166,35 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 z TableScan alias: z Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-5 Map Reduce @@ -195,60 +203,64 @@ STAGE PLANS: alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col5} + 0 {key} {value} 1 {value} keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col5, _col9 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col5} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col5, _col9 + Position of Big Table: 0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string #### A masked pattern was here #### - name default.dest_j1 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/join32_lessSize.q.out ql/src/test/results/clientpositive/join32_lessSize.q.out index 7162738..04ed6b2 100644 --- ql/src/test/results/clientpositive/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/join32_lessSize.q.out @@ -126,14 +126,18 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-6 Map Reduce @@ -142,35 +146,39 @@ STAGE PLANS: alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col5 - columns.types string,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col5 + columns.types string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -325,14 +333,18 @@ STAGE PLANS: alias: z Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-5 Map Reduce @@ -350,17 +362,17 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0, _col5, _col9 Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -711,14 +723,18 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {key} - keys: - 0 value (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-11 Map Reduce @@ -727,35 +743,39 @@ STAGE PLANS: alias: w Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {key} - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col4 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col4 - columns.types string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col4 + Position of Big Table: 0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col4 + columns.types string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -869,30 +889,36 @@ STAGE PLANS: TableScan alias: y GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col4} - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 0 z TableScan alias: z GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col4} - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 0 Stage: Stage-7 Map Reduce @@ -1120,16 +1146,19 @@ STAGE PLANS: TableScan alias: z GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 1 Stage: Stage-8 Map Reduce @@ -1137,53 +1166,56 @@ STAGE PLANS: TableScan alias: y GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {_col4} - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - outputColumnNames: _col4, _col9, _col13 - Position of Big Table: 1 - Select Operator - expressions: _col4 (type: string), _col13 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col4, _col9, _col13 + Position of Big Table: 1 + Select Operator + expressions: _col4 (type: string), _col13 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1327,16 +1359,19 @@ STAGE PLANS: TableScan alias: y GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 Stage: Stage-9 Map Reduce @@ -1344,53 +1379,56 @@ STAGE PLANS: TableScan alias: z GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {_col4} - 1 {value} - 2 {value} - keys: - 0 _col4 (type: string) - 1 key (type: string) - 2 key (type: string) - outputColumnNames: _col4, _col9, _col13 - Position of Big Table: 2 - Select Operator - expressions: _col4 (type: string), _col13 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col4, _col9, _col13 + Position of Big Table: 2 + Select Operator + expressions: _col4 (type: string), _col13 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -1516,30 +1554,38 @@ STAGE PLANS: key expressions: _col4 (type: string) sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE tag: 0 TableScan alias: z Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - tag: 2 - value expressions: value (type: string) + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + tag: 2 + value expressions: value (type: string) TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: value (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1666,17 +1712,17 @@ STAGE PLANS: 1 {VALUE._col0} 2 {VALUE._col0} outputColumnNames: _col4, _col9, _col13 - Statistics: Num rows: 138 Data size: 14064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 7031 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: string), _col13 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 138 Data size: 14064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 7031 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 138 Data size: 14064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 7031 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1942,14 +1988,18 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 Stage: Stage-7 Map Reduce @@ -1958,39 +2008,43 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Position of Big Table: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -2145,14 +2199,18 @@ STAGE PLANS: alias: z Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-6 Map Reduce @@ -2170,17 +2228,17 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0, _col1, _col3 Position of Big Table: 0 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -2558,39 +2616,43 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Left Outer Join0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 + Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -2745,14 +2807,18 @@ STAGE PLANS: alias: z Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-6 Map Reduce @@ -3071,13 +3137,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-7 Map Reduce @@ -3085,27 +3154,30 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -3120,13 +3192,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) Stage: Stage-6 Map Reduce @@ -3142,14 +3217,14 @@ STAGE PLANS: 0 _col1 (type: string) 1 value (type: string) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -3319,13 +3394,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-7 Map Reduce @@ -3333,27 +3411,30 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -3368,13 +3449,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) Stage: Stage-6 Map Reduce @@ -3390,14 +3474,14 @@ STAGE PLANS: 0 _col1 (type: string) 1 value (type: string) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join33.q.out ql/src/test/results/clientpositive/join33.q.out index 56024b6..ff0d7cc 100644 --- ql/src/test/results/clientpositive/join33.q.out +++ ql/src/test/results/clientpositive/join33.q.out @@ -166,27 +166,35 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 z TableScan alias: z Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 Stage: Stage-5 Map Reduce @@ -195,60 +203,64 @@ STAGE PLANS: alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col5} + 0 {key} {value} 1 {value} keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col5, _col9 - Position of Big Table: 0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col5} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col5, _col9 + Position of Big Table: 0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string #### A masked pattern was here #### - name default.dest_j1 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/join34.q.out ql/src/test/results/clientpositive/join34.q.out index 2c02bc5..b52777a 100644 --- ql/src/test/results/clientpositive/join34.q.out +++ ql/src/test/results/clientpositive/join34.q.out @@ -156,14 +156,18 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col1} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-5 Map Reduce @@ -174,14 +178,14 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 20) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2004 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -193,17 +197,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -230,14 +234,14 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2004 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -249,17 +253,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 19 Data size: 3966 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 2204 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/join35.q.out ql/src/test/results/clientpositive/join35.q.out index 2d69e3f..3053af7 100644 --- ql/src/test/results/clientpositive/join35.q.out +++ ql/src/test/results/clientpositive/join35.q.out @@ -163,23 +163,23 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 20) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) Path -> Alias: @@ -238,11 +238,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -273,14 +273,18 @@ STAGE PLANS: alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {_col1} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Position of Big Table: 0 Stage: Stage-6 Map Reduce @@ -288,7 +292,7 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -300,17 +304,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -334,7 +338,7 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 18 Data size: 1802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -346,17 +350,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -504,23 +508,23 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) Path -> Alias: @@ -579,11 +583,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 diff --git ql/src/test/results/clientpositive/join36.q.out ql/src/test/results/clientpositive/join36.q.out index 1f14642..7f76e35 100644 --- ql/src/test/results/clientpositive/join36.q.out +++ ql/src/test/results/clientpositive/join36.q.out @@ -80,13 +80,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {cnt} - 1 {cnt} - keys: - 0 key (type: int) - 1 key (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {cnt} + 1 {cnt} + keys: + 0 key (type: int) + 1 key (type: int) Stage: Stage-1 Map Reduce @@ -94,29 +97,32 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {cnt} - 1 {cnt} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 339 Data size: 1630 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 339 Data size: 1630 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 339 Data size: 1630 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {cnt} + 1 {cnt} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join37.q.out ql/src/test/results/clientpositive/join37.q.out index 184cf53..5e475d5 100644 --- ql/src/test/results/clientpositive/join37.q.out +++ ql/src/test/results/clientpositive/join37.q.out @@ -42,13 +42,16 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -56,29 +59,32 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join38.q.out ql/src/test/results/clientpositive/join38.q.out index e26333a..2c4882f 100644 --- ql/src/test/results/clientpositive/join38.q.out +++ ql/src/test/results/clientpositive/join38.q.out @@ -65,8 +65,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 111) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 111)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -82,8 +82,8 @@ STAGE PLANS: alias: b Statistics: Num rows: 2 Data size: 126 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (col11 = 111) (type: boolean) - Statistics: Num rows: 1 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (col11 is not null and (col11 = 111)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -94,22 +94,22 @@ STAGE PLANS: 0 key (type: string) 1 col11 (type: string) outputColumnNames: _col1, _col9 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col9 (type: string) outputColumnNames: _col1, _col9 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col1 (type: string), _col9 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work @@ -119,14 +119,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 1439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join40.q.out ql/src/test/results/clientpositive/join40.q.out index 0196f7a..69f6cbe 100644 --- ql/src/test/results/clientpositive/join40.q.out +++ ql/src/test/results/clientpositive/join40.q.out @@ -657,20 +657,26 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -679,14 +685,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -3729,19 +3735,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -3749,9 +3761,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash diff --git ql/src/test/results/clientpositive/join9.q.out ql/src/test/results/clientpositive/join9.q.out index 1240065..16ce5a2 100644 --- ql/src/test/results/clientpositive/join9.q.out +++ ql/src/test/results/clientpositive/join9.q.out @@ -80,23 +80,31 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: value (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -202,17 +210,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col7 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col7 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/join_alt_syntax.q.out ql/src/test/results/clientpositive/join_alt_syntax.q.out index 8299a2e..583343c 100644 --- ql/src/test/results/clientpositive/join_alt_syntax.q.out +++ ql/src/test/results/clientpositive/join_alt_syntax.q.out @@ -114,27 +114,36 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan alias: p2 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan alias: p1 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -145,17 +154,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} 2 {KEY.reducesinkkey0} outputColumnNames: _col1, _col12, _col23 - Statistics: Num rows: 68 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3601 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col1 = _col12) and (_col12 = _col23)) (type: boolean) - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col12 (type: string), _col23 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -186,31 +195,40 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan alias: p3 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan alias: p1 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -221,17 +239,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} 2 {KEY.reducesinkkey0} outputColumnNames: _col1, _col11, _col13 - Statistics: Num rows: 68 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3601 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col1 = _col11) and (_col11 = _col13)) (type: boolean) - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col11 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 823 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -264,10 +282,13 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_name (type: string) TableScan alias: p1 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE @@ -300,11 +321,14 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col12 (type: string) @@ -320,17 +344,17 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col11} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col11, _col12, _col23 - Statistics: Num rows: 34 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1861 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col11 + _col0) = _col0) and (_col23 = _col12)) (type: boolean) - Statistics: Num rows: 8 Data size: 821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 437 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col12 (type: string), _col23 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 437 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 437 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -365,19 +389,25 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE TableScan alias: p1 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -386,7 +416,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col11, _col12 - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 930 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -400,17 +430,20 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 930 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col11 (type: int) Reduce Operator Tree: Join Operator @@ -420,7 +453,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col11} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col11, _col12, _col23 - Statistics: Num rows: 36 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1800 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -434,18 +467,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 36 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1800 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col11 (type: int), _col12 (type: string), _col23 (type: string) Reduce Operator Tree: Join Operator @@ -455,17 +491,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col10} {VALUE._col11} {VALUE._col22} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col11, _col12, _col23, _col33, _col34 - Statistics: Num rows: 39 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 1980 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col12 = _col23) and (_col0 = _col33)) and (_col0 = _col11)) (type: boolean) - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col12 (type: string), _col23 (type: string), _col34 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -500,19 +536,25 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE TableScan alias: p1 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 8 Data size: 846 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -521,7 +563,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col11, _col12 - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 930 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -535,17 +577,20 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 31 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 16 Data size: 1637 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 930 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col11 (type: int) Reduce Operator Tree: Join Operator @@ -555,7 +600,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col11} {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col11, _col12, _col23 - Statistics: Num rows: 36 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1800 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -569,18 +614,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 36 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1800 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col11 (type: int), _col12 (type: string), _col23 (type: string) Reduce Operator Tree: Join Operator @@ -590,17 +638,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col10} {VALUE._col11} {VALUE._col22} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col11, _col12, _col23, _col33, _col34 - Statistics: Num rows: 39 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 1980 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col12 = _col23) and (_col0 = _col33)) and (_col0 = _col11)) (type: boolean) - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col12 (type: string), _col23 (type: string), _col34 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 433 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out index 06a58ae..7892eb8 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out @@ -56,30 +56,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -90,14 +99,14 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -126,30 +135,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -160,14 +178,14 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -198,10 +216,13 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE @@ -234,12 +255,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) @@ -255,14 +279,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -294,11 +318,11 @@ STAGE PLANS: alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_partkey = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + predicate: ((p_partkey = 1) and p_name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 634 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 634 Basic stats: COMPLETE Column stats: NONE value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 @@ -329,12 +353,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) diff --git ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out index ba90aae..99fe999 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out @@ -56,39 +56,51 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -101,14 +113,14 @@ STAGE PLANS: 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 3 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -141,21 +153,27 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -164,7 +182,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -178,18 +196,21 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string) Reduce Operator Tree: Join Operator @@ -199,7 +220,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -213,18 +234,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) Reduce Operator Tree: Join Operator @@ -234,14 +258,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col10} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col21} {VALUE._col22} {VALUE._col23} {VALUE._col24} {VALUE._col25} {VALUE._col26} {VALUE._col27} {VALUE._col28} {VALUE._col29} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2302 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2302 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2302 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out index 0ae4e77..086d145 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out @@ -58,30 +58,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -92,17 +101,17 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col1 = _col12) and (_col12 = _col23)) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -133,30 +142,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -167,17 +185,17 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col12 = _col1) and (_col23 = _col12)) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -210,10 +228,13 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE @@ -246,12 +267,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) @@ -267,17 +291,17 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col11 + _col0) = _col0) and (_col23 = _col12)) (type: boolean) - Statistics: Num rows: 1 Data size: 698 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 1 Data size: 698 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 698 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -311,11 +335,11 @@ STAGE PLANS: alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_partkey = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + predicate: (p_name is not null and (p_partkey = 1)) (type: boolean) + Statistics: Num rows: 1 Data size: 634 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 634 Basic stats: COMPLETE Column stats: NONE value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 @@ -346,12 +370,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) diff --git ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out index 4df0d7d..c79b452 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out @@ -58,39 +58,51 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -103,17 +115,17 @@ STAGE PLANS: 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 3 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col12 = _col23) and (_col1 = _col34)) (type: boolean) - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -148,21 +160,27 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -171,7 +189,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -185,18 +203,21 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string) Reduce Operator Tree: Join Operator @@ -206,7 +227,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -220,18 +241,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) Reduce Operator Tree: Join Operator @@ -241,7 +265,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col10} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col21} {VALUE._col22} {VALUE._col23} {VALUE._col24} {VALUE._col25} {VALUE._col26} {VALUE._col27} {VALUE._col28} {VALUE._col29} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2302 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col12 = _col23) and (_col0 = _col33)) and (_col0 = _col11)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out index 4ad0ffe..8c0da3d 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out @@ -110,30 +110,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -144,14 +153,14 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -180,30 +189,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -214,14 +232,14 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -252,10 +270,13 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_name (type: string), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_name (type: string), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE @@ -288,12 +309,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) @@ -348,7 +372,7 @@ STAGE PLANS: alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (p2_partkey = 1) (type: boolean) + predicate: ((p2_partkey = 1) and p2_name is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator sort order: @@ -383,12 +407,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out index e708327..dc4f966 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out @@ -110,39 +110,51 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -155,14 +167,14 @@ STAGE PLANS: 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 3 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -195,21 +207,27 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string), p2_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p2_name (type: string), p2_partkey (type: int) + Filter Operator + predicate: (p2_name is not null and p2_partkey is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string), p2_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p2_name (type: string), p2_partkey (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -218,7 +236,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -232,18 +250,21 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string) Reduce Operator Tree: Join Operator @@ -253,7 +274,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1534 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -267,18 +288,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1534 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) Reduce Operator Tree: Join Operator @@ -288,14 +312,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col10} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col21} {VALUE._col22} {VALUE._col23} {VALUE._col24} {VALUE._col25} {VALUE._col26} {VALUE._col27} {VALUE._col28} {VALUE._col29} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out index 05aeefe..82f5474 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out @@ -112,30 +112,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -146,17 +155,17 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col1 = _col12) and (_col12 = _col23)) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -187,30 +196,39 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -221,17 +239,17 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 11 Data size: 6980 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 4186 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col12 = _col1) and (_col23 = _col12)) (type: boolean) - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 697 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -264,10 +282,13 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_name (type: string), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_name (type: string), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE @@ -300,12 +321,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) @@ -365,7 +389,7 @@ STAGE PLANS: alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (p2_partkey = 1) (type: boolean) + predicate: (p2_name is not null and (p2_partkey = 1)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator sort order: @@ -400,12 +424,15 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out index 4bf17d2..8bbf616 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out @@ -112,39 +112,51 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string) - sort order: + - Map-reduce partition columns: p_name (type: string) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string) + sort order: + + Map-reduce partition columns: p_name (type: string) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string) - sort order: + - Map-reduce partition columns: p2_name (type: string) + Filter Operator + predicate: p2_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string) + sort order: + + Map-reduce partition columns: p2_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_partkey (type: int), p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -157,17 +169,17 @@ STAGE PLANS: 2 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} 3 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 16 Data size: 10470 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col12 = _col23) and (_col1 = _col34)) (type: boolean) - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string), _col33 (type: int), _col34 (type: string), _col35 (type: string), _col36 (type: string), _col37 (type: string), _col38 (type: int), _col39 (type: string), _col40 (type: double), _col41 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 2617 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -202,21 +214,27 @@ STAGE PLANS: TableScan alias: p2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p2_name (type: string), p2_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p2_name (type: string), p2_partkey (type: int) + Filter Operator + predicate: (p2_name is not null and p2_partkey is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) + Reduce Output Operator + key expressions: p2_name (type: string), p2_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p2_name (type: string), p2_partkey (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p2_mfgr (type: string), p2_brand (type: string), p2_type (type: string), p2_size (type: int), p2_container (type: string), p2_retailprice (type: double), p2_comment (type: string) TableScan alias: p1 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_name (type: string), p_partkey (type: int) - sort order: ++ - Map-reduce partition columns: p_name (type: string), p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_name (type: string), p_partkey (type: int) + sort order: ++ + Map-reduce partition columns: p_name (type: string), p_partkey (type: int) + Statistics: Num rows: 2 Data size: 1269 Basic stats: COMPLETE Column stats: NONE + value expressions: p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: Join Operator condition map: @@ -225,7 +243,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} 1 {KEY.reducesinkkey1} {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -239,18 +257,21 @@ STAGE PLANS: TableScan alias: p3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p3_name (type: string) - sort order: + - Map-reduce partition columns: p3_name (type: string) + Filter Operator + predicate: p3_name is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) + Reduce Output Operator + key expressions: p3_name (type: string) + sort order: + + Map-reduce partition columns: p3_name (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p3_partkey (type: int), p3_mfgr (type: string), p3_brand (type: string), p3_type (type: string), p3_size (type: int), p3_container (type: string), p3_retailprice (type: double), p3_comment (type: string) TableScan Reduce Output Operator key expressions: _col12 (type: string) sort order: + Map-reduce partition columns: _col12 (type: string) - Statistics: Num rows: 5 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1395 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string) Reduce Operator Tree: Join Operator @@ -260,7 +281,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {KEY.reducesinkkey0} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1534 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -274,18 +295,21 @@ STAGE PLANS: TableScan alias: p4 Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 5 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 3 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 3839 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 1534 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: int), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int), _col17 (type: string), _col18 (type: double), _col19 (type: string), _col22 (type: int), _col23 (type: string), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: double), _col30 (type: string) Reduce Operator Tree: Join Operator @@ -295,7 +319,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col10} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col21} {VALUE._col22} {VALUE._col23} {VALUE._col24} {VALUE._col25} {VALUE._col26} {VALUE._col27} {VALUE._col28} {VALUE._col29} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41 - Statistics: Num rows: 5 Data size: 4222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col12 = _col23) and (_col0 = _col33)) and (_col0 = _col11)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE diff --git ql/src/test/results/clientpositive/join_hive_626.q.out ql/src/test/results/clientpositive/join_hive_626.q.out index 2659597..2a0cd33 100644 --- ql/src/test/results/clientpositive/join_hive_626.q.out +++ ql/src/test/results/clientpositive/join_hive_626.q.out @@ -73,21 +73,27 @@ STAGE PLANS: TableScan alias: hive_foo Statistics: Num rows: 0 Data size: 15 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: foo_id (type: int) - sort order: + - Map-reduce partition columns: foo_id (type: int) - Statistics: Num rows: 0 Data size: 15 Basic stats: PARTIAL Column stats: NONE - value expressions: foo_name (type: string) + Filter Operator + predicate: foo_id is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: foo_id (type: int) + sort order: + + Map-reduce partition columns: foo_id (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: foo_name (type: string) TableScan alias: hive_bar Statistics: Num rows: 0 Data size: 23 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: foo_id (type: int) - sort order: + - Map-reduce partition columns: foo_id (type: int) - Statistics: Num rows: 0 Data size: 23 Basic stats: PARTIAL Column stats: NONE - value expressions: bar_id (type: int), bar_name (type: string) + Filter Operator + predicate: (foo_id is not null and bar_id is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: foo_id (type: int) + sort order: + + Map-reduce partition columns: foo_id (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: bar_id (type: int), bar_name (type: string) Reduce Operator Tree: Join Operator condition map: @@ -96,7 +102,7 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col0} {VALUE._col3} outputColumnNames: _col1, _col8, _col12 - Statistics: Num rows: 0 Data size: 16 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -112,17 +118,20 @@ STAGE PLANS: key expressions: _col8 (type: int) sort order: + Map-reduce partition columns: _col8 (type: int) - Statistics: Num rows: 0 Data size: 16 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: string), _col12 (type: string) TableScan alias: hive_count Statistics: Num rows: 0 Data size: 5 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: bar_id (type: int) - sort order: + - Map-reduce partition columns: bar_id (type: int) - Statistics: Num rows: 0 Data size: 5 Basic stats: PARTIAL Column stats: NONE - value expressions: n (type: int) + Filter Operator + predicate: bar_id is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: bar_id (type: int) + sort order: + + Map-reduce partition columns: bar_id (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: n (type: int) Reduce Operator Tree: Join Operator condition map: @@ -131,14 +140,14 @@ STAGE PLANS: 0 {VALUE._col1} {VALUE._col11} 1 {VALUE._col0} outputColumnNames: _col1, _col12, _col20 - Statistics: Num rows: 0 Data size: 17 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col1 (type: string), _col12 (type: string), _col20 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 0 Data size: 17 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 17 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_map_ppr.q.out ql/src/test/results/clientpositive/join_map_ppr.q.out index 1c5ea03..51fb6c6 100644 --- ql/src/test/results/clientpositive/join_map_ppr.q.out +++ ql/src/test/results/clientpositive/join_map_ppr.q.out @@ -129,31 +129,39 @@ STAGE PLANS: alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 y TableScan alias: y Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 Stage: Stage-1 Map Reduce @@ -162,51 +170,55 @@ STAGE PLANS: alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key} - 1 {value} - 2 {value} - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col5, _col9 - Position of Big Table: 2 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col5, _col9 + Position of Big Table: 2 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -734,31 +746,39 @@ STAGE PLANS: alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - 2 {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Position of Big Table: 2 y TableScan alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {value} - 2 {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - Position of Big Table: 2 + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Position of Big Table: 2 Stage: Stage-1 Map Reduce @@ -767,56 +787,60 @@ STAGE PLANS: alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key} - 1 {value} - 2 {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - 2 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col5, _col9 - Position of Big Table: 2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 107 - rawDataSize 2018 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2125 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col5, _col9 + Position of Big Table: 2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col9 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 107 + rawDataSize 2018 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2125 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/join_nullsafe.q.out ql/src/test/results/clientpositive/join_nullsafe.q.out index 360d9c8..a729af7 100644 --- ql/src/test/results/clientpositive/join_nullsafe.q.out +++ ql/src/test/results/clientpositive/join_nullsafe.q.out @@ -111,30 +111,39 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) TableScan alias: c Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) TableScan alias: a Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) Reduce Operator Tree: Join Operator condition map: @@ -145,14 +154,14 @@ STAGE PLANS: 1 {VALUE._col0} {KEY.reducesinkkey0} 2 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -295,27 +304,36 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int), key (type: int) - sort order: ++ - Map-reduce partition columns: value (type: int), key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int), key (type: int) + sort order: ++ + Map-reduce partition columns: value (type: int), key (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: c Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: int) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: int) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: int) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: int) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: int) + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -327,14 +345,14 @@ STAGE PLANS: 2 {KEY.reducesinkkey0} {KEY.reducesinkkey1} nullSafes: [true, false] outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 37 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_rc.q.out ql/src/test/results/clientpositive/join_rc.q.out index be7ca85..537865f 100644 --- ql/src/test/results/clientpositive/join_rc.q.out +++ ql/src/test/results/clientpositive/join_rc.q.out @@ -51,20 +51,26 @@ STAGE PLANS: TableScan alias: join_rc2 Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: join_rc1 Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -73,14 +79,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 550 Data size: 5293 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2646 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_reorder.q.out ql/src/test/results/clientpositive/join_reorder.q.out index 4fc41e0..caef085 100644 --- ql/src/test/results/clientpositive/join_reorder.q.out +++ ql/src/test/results/clientpositive/join_reorder.q.out @@ -60,21 +60,27 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -83,14 +89,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -119,21 +125,27 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -142,14 +154,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_reorder2.q.out ql/src/test/results/clientpositive/join_reorder2.q.out index cd8233c..7a8a167 100644 --- ql/src/test/results/clientpositive/join_reorder2.q.out +++ ql/src/test/results/clientpositive/join_reorder2.q.out @@ -81,39 +81,51 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -126,14 +138,14 @@ STAGE PLANS: 2 {KEY.reducesinkkey0} {VALUE._col0} 3 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -191,21 +203,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: (key is not null and (key + 1) is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: ((key is not null and val is not null) and (key + 1) is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -214,7 +232,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -230,17 +248,20 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: val (type: string) - sort order: + - Map-reduce partition columns: val (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: val is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val (type: string) + sort order: + + Map-reduce partition columns: val (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -249,7 +270,7 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col3} {VALUE._col4} 1 {VALUE._col0} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -263,18 +284,21 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan Reduce Output Operator key expressions: (_col0 + 1) (type: double) sort order: + Map-reduce partition columns: (_col0 + 1) (type: double) - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) Reduce Operator Tree: Join Operator @@ -284,14 +308,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col4} {VALUE._col5} {VALUE._col8} {VALUE._col9} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_reorder3.q.out ql/src/test/results/clientpositive/join_reorder3.q.out index 40c2951..6a8c3ea 100644 --- ql/src/test/results/clientpositive/join_reorder3.q.out +++ ql/src/test/results/clientpositive/join_reorder3.q.out @@ -81,39 +81,51 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -126,14 +138,14 @@ STAGE PLANS: 2 {KEY.reducesinkkey0} {VALUE._col0} 3 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -191,21 +203,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: (key is not null and (key + 1) is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: ((key is not null and val is not null) and (key + 1) is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -214,7 +232,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -230,17 +248,20 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: val (type: string) - sort order: + - Map-reduce partition columns: val (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: val is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val (type: string) + sort order: + + Map-reduce partition columns: val (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -249,7 +270,7 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col3} {VALUE._col4} 1 {VALUE._col0} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -263,18 +284,21 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan Reduce Output Operator key expressions: (_col0 + 1) (type: double) sort order: + Map-reduce partition columns: (_col0 + 1) (type: double) - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) Reduce Operator Tree: Join Operator @@ -284,14 +308,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {VALUE._col4} {VALUE._col5} {VALUE._col8} {VALUE._col9} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_reorder4.q.out ql/src/test/results/clientpositive/join_reorder4.q.out index 31faa58..93c783d 100644 --- ql/src/test/results/clientpositive/join_reorder4.q.out +++ ql/src/test/results/clientpositive/join_reorder4.q.out @@ -67,28 +67,34 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) c TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key3 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) Stage: Stage-4 Map Reduce @@ -96,31 +102,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key1} {val1} - 1 {key2} {val2} - 2 {key3} {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -167,28 +176,34 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) c TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key3 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) Stage: Stage-4 Map Reduce @@ -196,31 +211,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key1} {val1} - 1 {key2} {val2} - 2 {key3} {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -267,28 +285,34 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) c TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {val1} - 1 {key2} {val2} - 2 {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) + Filter Operator + predicate: key3 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {val1} + 1 {key2} {val2} + 2 {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) Stage: Stage-4 Map Reduce @@ -296,31 +320,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key1} {val1} - 1 {key2} {val2} - 2 {key3} {val3} - keys: - 0 key1 (type: string) - 1 key2 (type: string) - 2 key3 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join_star.q.out ql/src/test/results/clientpositive/join_star.q.out index 4e2158a..f63c1f8 100644 --- ql/src/test/results/clientpositive/join_star.q.out +++ ql/src/test/results/clientpositive/join_star.q.out @@ -139,13 +139,16 @@ STAGE PLANS: TableScan alias: dim1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {m1} {m2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) + Filter Operator + predicate: f1 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) Stage: Stage-3 Map Reduce @@ -153,28 +156,31 @@ STAGE PLANS: TableScan alias: fact Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {m1} {m2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: d1 is not null (type: boolean) + Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -228,24 +234,30 @@ STAGE PLANS: TableScan alias: dim1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {m1} {m2} {d2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) + Filter Operator + predicate: f1 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} {d2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) dim2 TableScan alias: dim2 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col7} - 1 {f4} - keys: - 0 _col3 (type: int) - 1 f3 (type: int) + Filter Operator + predicate: f3 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col7} + 1 {f4} + keys: + 0 _col3 (type: int) + 1 f3 (type: int) Stage: Stage-4 Map Reduce @@ -253,39 +265,42 @@ STAGE PLANS: TableScan alias: fact Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {m1} {m2} {d2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) - outputColumnNames: _col0, _col1, _col3, _col7 - Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (d1 is not null and d2 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col7} - 1 {f4} + 0 {m1} {m2} {d2} + 1 {f2} keys: - 0 _col3 (type: int) - 1 f3 (type: int) - outputColumnNames: _col0, _col1, _col7, _col11 - Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 d1 (type: int) + 1 f1 (type: int) + outputColumnNames: _col0, _col1, _col3, _col7 + Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col7} + 1 {f4} + keys: + 0 _col3 (type: int) + 1 f3 (type: int) + outputColumnNames: _col0, _col1, _col7, _col11 + Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -341,24 +356,30 @@ STAGE PLANS: TableScan alias: dim1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {m1} {m2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) + Filter Operator + predicate: (f1 is not null and f2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) dim2 TableScan alias: dim2 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col7} - 1 {f4} - keys: - 0 _col7 (type: int) - 1 f3 (type: int) + Filter Operator + predicate: f3 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col7} + 1 {f4} + keys: + 0 _col7 (type: int) + 1 f3 (type: int) Stage: Stage-4 Map Reduce @@ -366,39 +387,42 @@ STAGE PLANS: TableScan alias: fact Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {m1} {m2} - 1 {f2} - keys: - 0 d1 (type: int) - 1 f1 (type: int) - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: d1 is not null (type: boolean) + Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col7} - 1 {f4} + 0 {m1} {m2} + 1 {f2} keys: - 0 _col7 (type: int) - 1 f3 (type: int) - outputColumnNames: _col0, _col1, _col7, _col11 - Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 d1 (type: int) + 1 f1 (type: int) + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col7} + 1 {f4} + keys: + 0 _col7 (type: int) + 1 f3 (type: int) + outputColumnNames: _col0, _col1, _col7, _col11 + Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col7 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join_thrift.q.out ql/src/test/results/clientpositive/join_thrift.q.out index 7869a7e..67a6966 100644 --- ql/src/test/results/clientpositive/join_thrift.q.out +++ ql/src/test/results/clientpositive/join_thrift.q.out @@ -33,20 +33,26 @@ STAGE PLANS: TableScan alias: s2 Statistics: Num rows: 0 Data size: 1606 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: aint (type: int) - sort order: + - Map-reduce partition columns: aint (type: int) - Statistics: Num rows: 0 Data size: 1606 Basic stats: PARTIAL Column stats: NONE - value expressions: lintstring (type: array>) + Filter Operator + predicate: aint is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: aint (type: int) + sort order: + + Map-reduce partition columns: aint (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: lintstring (type: array>) TableScan alias: s1 Statistics: Num rows: 401 Data size: 1606 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: aint (type: int) - sort order: + - Map-reduce partition columns: aint (type: int) - Statistics: Num rows: 401 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: aint is not null (type: boolean) + Statistics: Num rows: 201 Data size: 805 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: aint (type: int) + sort order: + + Map-reduce partition columns: aint (type: int) + Statistics: Num rows: 201 Data size: 805 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -55,14 +61,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col3} outputColumnNames: _col0, _col12 - Statistics: Num rows: 441 Data size: 1766 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 221 Data size: 885 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col12 (type: array>) outputColumnNames: _col0, _col1 - Statistics: Num rows: 441 Data size: 1766 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 221 Data size: 885 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 441 Data size: 1766 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 221 Data size: 885 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/join_vc.q.out ql/src/test/results/clientpositive/join_vc.q.out index bf21ceb..1a47f66 100644 --- ql/src/test/results/clientpositive/join_vc.q.out +++ ql/src/test/results/clientpositive/join_vc.q.out @@ -19,20 +19,26 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: t1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -41,7 +47,7 @@ STAGE PLANS: 0 1 {VALUE._col0} outputColumnNames: _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -55,18 +61,21 @@ STAGE PLANS: TableScan alias: t3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) TableScan Reduce Output Operator key expressions: _col5 (type: string) sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -75,11 +84,11 @@ STAGE PLANS: 0 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} outputColumnNames: _col8, _col9, _col10 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col10 (type: bigint), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -94,18 +103,18 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string) sort order: +++ - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 303 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/lateral_view_cp.q.out ql/src/test/results/clientpositive/lateral_view_cp.q.out index decb207..3afef7b 100644 --- ql/src/test/results/clientpositive/lateral_view_cp.q.out +++ ql/src/test/results/clientpositive/lateral_view_cp.q.out @@ -39,20 +39,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: array) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: array) Reduce Operator Tree: Join Operator condition map: @@ -61,22 +67,22 @@ STAGE PLANS: 0 1 {VALUE._col0} outputColumnNames: _col5 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: array) outputColumnNames: _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) mode: hash @@ -91,17 +97,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: array) outputColumnNames: _col0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) mode: hash diff --git ql/src/test/results/clientpositive/lateral_view_noalias.q.out ql/src/test/results/clientpositive/lateral_view_noalias.q.out index ff8348c..d51b2de 100644 --- ql/src/test/results/clientpositive/lateral_view_noalias.q.out +++ ql/src/test/results/clientpositive/lateral_view_noalias.q.out @@ -199,12 +199,15 @@ STAGE PLANS: Limit Number of rows: 2 Statistics: Num rows: 0 Data size: 11624 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -214,14 +217,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 11624 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE value expressions: _col1 (type: int) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 11624 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE value expressions: _col1 (type: int) Reduce Operator Tree: Join Operator @@ -231,14 +234,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 12786 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 12786 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 12786 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -297,12 +300,15 @@ STAGE PLANS: Limit Number of rows: 2 Statistics: Num rows: 0 Data size: 11624 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/limit_pushdown.q.out ql/src/test/results/clientpositive/limit_pushdown.q.out index 01571d0..75d21f1 100644 --- ql/src/test/results/clientpositive/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/limit_pushdown.q.out @@ -806,12 +806,15 @@ STAGE PLANS: Limit Number of rows: 2 Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -821,14 +824,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -838,17 +841,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 4 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -915,12 +918,15 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/limit_pushdown_negative.q.out ql/src/test/results/clientpositive/limit_pushdown_negative.q.out index 93ec6a6..f5b92b4 100644 --- ql/src/test/results/clientpositive/limit_pushdown_negative.q.out +++ ql/src/test/results/clientpositive/limit_pushdown_negative.q.out @@ -15,21 +15,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -38,17 +44,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 - Statistics: Num rows: 20 Data size: 4120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 4120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/mapjoin_distinct.q.out ql/src/test/results/clientpositive/mapjoin_distinct.q.out index 1b7bddf..0737fcd 100644 --- ql/src/test/results/clientpositive/mapjoin_distinct.q.out +++ ql/src/test/results/clientpositive/mapjoin_distinct.q.out @@ -28,13 +28,16 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -42,31 +45,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: rand() (type: double) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: rand() (type: double) + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -74,7 +80,7 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: partials outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -90,20 +96,20 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: final outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -172,13 +178,16 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -186,31 +195,34 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -218,14 +230,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -295,13 +307,16 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -309,26 +324,29 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: rand() (type: double) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: rand() (type: double) + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -336,7 +354,7 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: partial1 outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -352,20 +370,20 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: final outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -434,13 +452,16 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -448,26 +469,29 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -475,14 +499,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6342 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out index a2a293c..567b0ca 100644 --- ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out @@ -26,24 +26,30 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 value (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 value (type: string) + 1 value (type: string) src1 TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -51,39 +57,42 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} + 0 {key} 1 keys: - 0 _col0 (type: string) - 1 key (type: string) + 0 value (type: string) + 1 value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -120,8 +129,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} @@ -133,13 +142,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -148,8 +160,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 38 Data size: 7615 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and key is not null) and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -160,7 +172,7 @@ STAGE PLANS: 0 value (type: string) 1 value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 41 Data size: 8376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -171,14 +183,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -217,13 +229,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} {ds} - 1 - keys: - 0 value (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {ds} + 1 + keys: + 0 value (type: string) + 1 value (type: string) Stage: Stage-7 Map Reduce @@ -231,23 +246,26 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {ds} - 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {ds} + 1 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -262,13 +280,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col2} - 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col2} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -284,22 +305,22 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col2 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work @@ -309,14 +330,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/mapjoin_memcheck.q.out ql/src/test/results/clientpositive/mapjoin_memcheck.q.out index 135e979..36abdc7 100644 --- ql/src/test/results/clientpositive/mapjoin_memcheck.q.out +++ ql/src/test/results/clientpositive/mapjoin_memcheck.q.out @@ -44,13 +44,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -58,28 +61,31 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/mapjoin_subquery.q.out ql/src/test/results/clientpositive/mapjoin_subquery.q.out index c372037..7838eed 100644 --- ql/src/test/results/clientpositive/mapjoin_subquery.q.out +++ ql/src/test/results/clientpositive/mapjoin_subquery.q.out @@ -38,24 +38,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) z TableScan alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -63,43 +69,46 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -272,24 +281,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) z TableScan alias: z Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -297,43 +312,46 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 34 Data size: 3515 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/mapjoin_subquery2.q.out ql/src/test/results/clientpositive/mapjoin_subquery2.q.out index 54533b3..eebcaa2 100644 --- ql/src/test/results/clientpositive/mapjoin_subquery2.q.out +++ ql/src/test/results/clientpositive/mapjoin_subquery2.q.out @@ -99,24 +99,30 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {name} - 1 {name} {id} - keys: - 0 id (type: int) - 1 id (type: int) + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {name} + 1 {name} {id} + keys: + 0 id (type: int) + 1 id (type: int) z TableScan alias: z Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} - 1 {name} - keys: - 0 _col0 (type: int) - 1 id (type: int) + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} + 1 {name} + keys: + 0 _col0 (type: int) + 1 id (type: int) Stage: Stage-5 Map Reduce @@ -124,43 +130,46 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 0 Data size: 13 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {id} {name} - 1 {name} {id} - keys: - 0 id (type: int) - 1 id (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col5 (type: int), _col4 (type: string), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} - 1 {id} {name} - keys: - 0 _col0 (type: int) - 1 id (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: int), _col5 (type: string) + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {id} {name} + 1 {name} {id} + keys: + 0 id (type: int) + 1 id (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col5 (type: int), _col4 (type: string), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} + 1 {id} {name} + keys: + 0 _col0 (type: int) + 1 id (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/mergejoins.q.out ql/src/test/results/clientpositive/mergejoins.q.out index 88af8bb..4e09572 100644 --- ql/src/test/results/clientpositive/mergejoins.q.out +++ ql/src/test/results/clientpositive/mergejoins.q.out @@ -49,39 +49,51 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: val1 (type: int) - sort order: + - Map-reduce partition columns: val1 (type: int) + Filter Operator + predicate: val1 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: val2 (type: int) + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val2 (type: int) TableScan alias: b Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: val1 (type: int) - sort order: + - Map-reduce partition columns: val1 (type: int) + Filter Operator + predicate: val1 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: val2 (type: int) + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val2 (type: int) TableScan alias: c Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: val1 (type: int) - sort order: + - Map-reduce partition columns: val1 (type: int) + Filter Operator + predicate: val1 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: val2 (type: int) + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val2 (type: int) TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: val1 (type: int) - sort order: + - Map-reduce partition columns: val1 (type: int) + Filter Operator + predicate: (val1 is not null and val2 is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: val2 (type: int) + Reduce Output Operator + key expressions: val1 (type: int) + sort order: + + Map-reduce partition columns: val1 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val2 (type: int) Reduce Operator Tree: Join Operator condition map: @@ -115,12 +127,15 @@ STAGE PLANS: TableScan alias: e Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: val2 (type: int) - sort order: + - Map-reduce partition columns: val2 (type: int) + Filter Operator + predicate: val2 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: val1 (type: int) + Reduce Output Operator + key expressions: val2 (type: int) + sort order: + + Map-reduce partition columns: val2 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val1 (type: int) Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/metadataonly1.q.out ql/src/test/results/clientpositive/metadataonly1.q.out index 2c9688e..f08ef9e 100644 --- ql/src/test/results/clientpositive/metadataonly1.q.out +++ ql/src/test/results/clientpositive/metadataonly1.q.out @@ -704,27 +704,31 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 + Filter Operator + isSamplingPred: false + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -735,7 +739,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE tag: 1 TableScan alias: a2 @@ -862,9 +866,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/multiMapJoin1.q.out ql/src/test/results/clientpositive/multiMapJoin1.q.out index cce4ba4..39e9712 100644 --- ql/src/test/results/clientpositive/multiMapJoin1.q.out +++ ql/src/test/results/clientpositive/multiMapJoin1.q.out @@ -195,24 +195,30 @@ STAGE PLANS: TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) smalltbl2 TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) Stage: Stage-2 Map Reduce @@ -220,42 +226,45 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 53120 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -351,24 +360,30 @@ STAGE PLANS: TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) smalltbl2 TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) Stage: Stage-2 Map Reduce @@ -376,42 +391,45 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 53120 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col1 (type: string) - 1 value (type: string) - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -511,24 +529,30 @@ STAGE PLANS: TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) smalltbl2 TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {key} - keys: - 0 _col1 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 _col1 (type: string) + 1 value (type: string) Stage: Stage-2 Map Reduce @@ -536,48 +560,51 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 53120 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {key} - keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col3 - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string) + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1375 Data size: 14608 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {key} + keys: + 0 _col1 (type: string) + 1 value (type: string) outputColumnNames: _col3 - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col3 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: string) + outputColumnNames: _col3 + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1512 Data size: 16068 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -586,14 +613,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 8034 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 8034 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 8034 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -826,38 +853,42 @@ STAGE PLANS: join3:join2:join1:smalltbl1 TableScan alias: smalltbl1 - HashTable Sink Operator - condition expressions: - 0 {key1} {key2} {value} - 1 - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 + keys: + 0 key1 (type: string) + 1 key (type: string) Stage: Stage-18 Map Reduce Map Operator Tree: TableScan alias: bigtbl - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -874,13 +905,15 @@ STAGE PLANS: join3:join2:smalltbl2 TableScan alias: smalltbl2 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) Stage: Stage-15 Map Reduce @@ -890,7 +923,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} + 0 {_col0} {_col1} {_col2} {_col3} {_col4} 1 {key} keys: 0 _col3 (type: string) @@ -921,13 +954,15 @@ STAGE PLANS: join3:smalltbl3 TableScan alias: smalltbl3 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 key (type: string) Stage: Stage-12 Map Reduce @@ -968,13 +1003,15 @@ STAGE PLANS: smalltbl4 TableScan alias: smalltbl4 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 - keys: - 0 _col2 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 + keys: + 0 _col2 (type: string) + 1 key (type: string) Stage: Stage-9 Map Reduce @@ -1054,29 +1091,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl4 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 {key} - keys: - 0 _col2 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Group By Operator - aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1088,16 +1127,19 @@ STAGE PLANS: key expressions: _col2 (type: string) sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) TableScan alias: smalltbl4 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1106,11 +1148,11 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) mode: hash @@ -1145,25 +1187,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl3 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 {key} - keys: - 0 _col1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1175,16 +1219,19 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: smalltbl3 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1193,11 +1240,11 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1216,7 +1263,7 @@ STAGE PLANS: TableScan HashTable Sink Operator condition expressions: - 0 {_col0} {_col1} {_col2} + 0 {_col0} {_col1} {_col2} {_col4} 1 {key} keys: 0 _col3 (type: string) @@ -1227,25 +1274,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1255,32 +1304,35 @@ STAGE PLANS: TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) TableScan Reduce Output Operator key expressions: _col3 (type: string) sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {KEY.reducesinkkey0} {KEY.reducesinkkey0} + 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {KEY.reducesinkkey0} {VALUE._col3} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1298,38 +1350,42 @@ STAGE PLANS: join3:join2:join1:bigtbl TableScan alias: bigtbl - HashTable Sink Operator - condition expressions: - 0 {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) Stage: Stage-19 Map Reduce Map Operator Tree: TableScan alias: smalltbl1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1339,20 +1395,26 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key1 (type: string) - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - value expressions: key2 (type: string), value (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key1 (type: string) + sort order: + + Map-reduce partition columns: key1 (type: string) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + value expressions: key2 (type: string), value (type: string) TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1361,11 +1423,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1567,46 +1629,58 @@ STAGE PLANS: TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key1} {key2} {value} - 1 - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 + keys: + 0 key1 (type: string) + 1 key (type: string) join3:join2:smalltbl2 TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) join3:smalltbl3 TableScan alias: smalltbl3 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 key (type: string) smalltbl4 TableScan alias: smalltbl4 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 - keys: - 0 _col2 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 + keys: + 0 _col2 (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -1614,75 +1688,78 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 {key} - keys: - 0 _col1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 {key} - keys: - 0 _col2 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1884,24 +1961,30 @@ STAGE PLANS: TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key1} {key2} {value} - 1 - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 + keys: + 0 key1 (type: string) + 1 key (type: string) join3:join2:smalltbl2 TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) Stage: Stage-11 Map Reduce @@ -1909,42 +1992,45 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1962,24 +2048,30 @@ STAGE PLANS: TableScan alias: smalltbl3 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 key (type: string) smalltbl4 TableScan alias: smalltbl4 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 - keys: - 0 _col2 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 + keys: + 0 _col2 (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -1995,11 +2087,11 @@ STAGE PLANS: 0 _col1 (type: string) 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -2010,11 +2102,11 @@ STAGE PLANS: 0 _col2 (type: string) 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) mode: hash @@ -2243,38 +2335,42 @@ STAGE PLANS: join3:join2:join1:smalltbl1 TableScan alias: smalltbl1 - HashTable Sink Operator - condition expressions: - 0 {key1} {key2} {value} - 1 - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 + keys: + 0 key1 (type: string) + 1 key (type: string) Stage: Stage-18 Map Reduce Map Operator Tree: TableScan alias: bigtbl - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -2291,13 +2387,15 @@ STAGE PLANS: join3:join2:smalltbl2 TableScan alias: smalltbl2 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) + Filter Operator + predicate: value is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) Stage: Stage-15 Map Reduce @@ -2307,7 +2405,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} + 0 {_col0} {_col1} {_col2} {_col3} {_col4} 1 {key} keys: 0 _col3 (type: string) @@ -2338,13 +2436,15 @@ STAGE PLANS: join3:smalltbl3 TableScan alias: smalltbl3 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 - keys: - 0 _col1 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 + keys: + 0 _col1 (type: string) + 1 key (type: string) Stage: Stage-12 Map Reduce @@ -2385,13 +2485,15 @@ STAGE PLANS: smalltbl4 TableScan alias: smalltbl4 - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 - keys: - 0 _col2 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 + keys: + 0 _col2 (type: string) + 1 key (type: string) Stage: Stage-9 Map Reduce @@ -2471,29 +2573,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl4 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} - 1 {key} - keys: - 0 _col2 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Group By Operator - aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -2505,16 +2609,19 @@ STAGE PLANS: key expressions: _col2 (type: string) sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) TableScan alias: smalltbl4 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2523,11 +2630,11 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col7 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col5, _col6 - Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 913 Data size: 13207 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col2)), sum(hash(_col3)), sum(hash(_col4)), sum(hash(_col7)), sum(hash(_col5)), sum(hash(_col6)) mode: hash @@ -2562,25 +2669,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl3 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} - 1 {key} - keys: - 0 _col1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -2592,16 +2701,19 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: smalltbl3 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2610,11 +2722,11 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 830 Data size: 12007 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2633,7 +2745,7 @@ STAGE PLANS: TableScan HashTable Sink Operator condition expressions: - 0 {_col0} {_col1} {_col2} + 0 {_col0} {_col1} {_col2} {_col4} 1 {key} keys: 0 _col3 (type: string) @@ -2644,25 +2756,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: smalltbl2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} {_col2} {_col3} {_col3} - 1 {key} - keys: - 0 _col3 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -2672,32 +2786,35 @@ STAGE PLANS: TableScan alias: smalltbl2 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) TableScan Reduce Output Operator key expressions: _col3 (type: string) sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {KEY.reducesinkkey0} {KEY.reducesinkkey0} + 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {KEY.reducesinkkey0} {VALUE._col3} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 755 Data size: 10916 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2715,38 +2832,42 @@ STAGE PLANS: join3:join2:join1:bigtbl TableScan alias: bigtbl - HashTable Sink Operator - condition expressions: - 0 {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + HashTable Sink Operator + condition expressions: + 0 {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) Stage: Stage-19 Map Reduce Map Operator Tree: TableScan alias: smalltbl1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key1} {key2} {value} - 1 {key} - keys: - 0 key1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col2, _col5 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -2756,20 +2877,26 @@ STAGE PLANS: TableScan alias: bigtbl Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key1 (type: string) - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 5000 Data size: 72180 Basic stats: COMPLETE Column stats: NONE - value expressions: key2 (type: string), value (type: string) + Filter Operator + predicate: ((key1 is not null and value is not null) and key2 is not null) (type: boolean) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key1 (type: string) + sort order: + + Map-reduce partition columns: key1 (type: string) + Statistics: Num rows: 625 Data size: 9022 Basic stats: COMPLETE Column stats: NONE + value expressions: key2 (type: string), value (type: string) TableScan alias: smalltbl1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -2778,11 +2905,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col2 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 687 Data size: 9924 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/multiMapJoin2.q.out ql/src/test/results/clientpositive/multiMapJoin2.q.out index 0481280..bb1d3db 100644 --- ql/src/test/results/clientpositive/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/multiMapJoin2.q.out @@ -34,24 +34,30 @@ STAGE PLANS: TableScan alias: y1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) null-subquery2:tmp-subquery2:y2 TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -59,69 +65,75 @@ STAGE PLANS: TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE TableScan alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -274,13 +286,16 @@ STAGE PLANS: TableScan alias: y1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-7 Map Reduce @@ -288,27 +303,30 @@ STAGE PLANS: TableScan alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -323,66 +341,72 @@ STAGE PLANS: TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce Map Operator Tree: TableScan Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -567,66 +591,72 @@ STAGE PLANS: TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce Map Operator Tree: TableScan Union - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -777,13 +807,16 @@ STAGE PLANS: TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -791,31 +824,34 @@ STAGE PLANS: TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -823,11 +859,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -894,7 +930,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -902,11 +938,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -921,15 +957,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -991,13 +1027,13 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1006,17 +1042,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1035,13 +1071,16 @@ STAGE PLANS: TableScan alias: y1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-8 Map Reduce @@ -1049,31 +1088,34 @@ STAGE PLANS: TableScan alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1081,11 +1123,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1204,24 +1246,30 @@ STAGE PLANS: TableScan alias: y1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) tmp2:y2 TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -1229,75 +1277,81 @@ STAGE PLANS: TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE TableScan alias: x1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1332,13 +1386,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -1502,30 +1556,33 @@ STAGE PLANS: TableScan alias: x1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -1592,7 +1649,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1600,11 +1657,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1619,15 +1676,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1729 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 850 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1689,13 +1746,13 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1704,17 +1761,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1733,13 +1790,16 @@ STAGE PLANS: TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -1747,31 +1807,34 @@ STAGE PLANS: TableScan alias: x2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1779,11 +1842,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1898,13 +1961,16 @@ STAGE PLANS: TableScan alias: y2 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -1912,64 +1978,70 @@ STAGE PLANS: TableScan alias: x1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: x2 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string) + keys: key (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x2 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Demux Operator - Statistics: Num rows: 65 Data size: 6609 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1652 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1652 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 64 Data size: 6506 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2004,13 +2076,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1652 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1652 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 64 Data size: 6506 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 3304 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -2180,13 +2252,16 @@ STAGE PLANS: TableScan alias: y Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -2194,27 +2269,30 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 125 Data size: 1261 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) - Statistics: Num rows: 137 Data size: 1387 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 137 Data size: 1387 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 63 Data size: 635 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Statistics: Num rows: 69 Data size: 698 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 69 Data size: 698 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2307,13 +2385,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-2 Map Reduce @@ -2321,31 +2402,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2353,11 +2437,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2379,24 +2463,30 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) null-subquery2:x-subquery2:c TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -2412,20 +2502,20 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2441,20 +2531,20 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 6392 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2473,13 +2563,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-7 Map Reduce @@ -2487,31 +2580,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -2519,11 +2615,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/multi_join_union.q.out ql/src/test/results/clientpositive/multi_join_union.q.out index a6d0eef..62e955e 100644 --- ql/src/test/results/clientpositive/multi_join_union.q.out +++ ql/src/test/results/clientpositive/multi_join_union.q.out @@ -59,55 +59,64 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) c-subquery1:a-subquery1:src13 TableScan alias: src13 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col4} {_col5} - 1 {_col0} - keys: - 0 _col5 (type: string) - 1 _col1 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col4} {_col5} + 1 {_col0} + keys: + 0 _col5 (type: string) + 1 _col1 (type: string) c-subquery2:a-subquery2:src14 TableScan alias: src14 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} {_col4} {_col5} - 1 {_col0} - keys: - 0 _col5 (type: string) - 1 _col1 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col4} {_col5} + 1 {_col0} + keys: + 0 _col5 (type: string) + 1 _col1 (type: string) Stage: Stage-6 Map Reduce @@ -115,39 +124,42 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} {_col4} {_col5} - 1 {_col0} {_col1} + 0 {key} {value} + 1 {key} {value} keys: - 0 _col5 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} {_col4} {_col5} + 1 {_col0} {_col1} + keys: + 0 _col5 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/nonmr_fetch.q.out ql/src/test/results/clientpositive/nonmr_fetch.q.out index bcc1399..5a13e84 100644 --- ql/src/test/results/clientpositive/nonmr_fetch.q.out +++ ql/src/test/results/clientpositive/nonmr_fetch.q.out @@ -1092,21 +1092,27 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1115,14 +1121,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/parquet_columnar.q.out ql/src/test/results/clientpositive/parquet_columnar.q.out index 5ccf37a..2222ff9 100644 --- ql/src/test/results/clientpositive/parquet_columnar.q.out +++ ql/src/test/results/clientpositive/parquet_columnar.q.out @@ -69,9 +69,6 @@ POSTHOOK: query: SELECT * FROM parquet_columnar_access POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_columnar_access #### A masked pattern was here #### -POSTHOOK: Lineage: parquet_columnar_access.f SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:f, type:float, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.i SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:i, type:int, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.s SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:s, type:string, comment:null), ] 1abc00 1 1.0 1def01 2 1.1 1ghi02 3 1.2 @@ -101,9 +98,6 @@ POSTHOOK: query: ALTER TABLE parquet_columnar_access REPLACE COLUMNS (s1 string, POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@parquet_columnar_access POSTHOOK: Output: default@parquet_columnar_access -POSTHOOK: Lineage: parquet_columnar_access.f SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:f, type:float, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.i SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:i, type:int, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.s SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:s, type:string, comment:null), ] PREHOOK: query: SELECT * FROM parquet_columnar_access PREHOOK: type: QUERY PREHOOK: Input: default@parquet_columnar_access @@ -112,9 +106,6 @@ POSTHOOK: query: SELECT * FROM parquet_columnar_access POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_columnar_access #### A masked pattern was here #### -POSTHOOK: Lineage: parquet_columnar_access.f SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:f, type:float, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.i SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:i, type:int, comment:null), ] -POSTHOOK: Lineage: parquet_columnar_access.s SIMPLE [(parquet_columnar_access_stage)parquet_columnar_access_stage.FieldSchema(name:s, type:string, comment:null), ] 1abc00 1 1.0 1def01 2 1.1 1ghi02 3 1.2 diff --git ql/src/test/results/clientpositive/pcr.q.out ql/src/test/results/clientpositive/pcr.q.out index 0894066..b74dfbf 100644 --- ql/src/test/results/clientpositive/pcr.q.out +++ ql/src/test/results/clientpositive/pcr.q.out @@ -2687,24 +2687,32 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: value (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string), ds (type: string) TableScan alias: t1 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: value (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string), ds (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2764,11 +2772,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -2795,7 +2803,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int) sort order: + - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) Path -> Alias: @@ -2828,13 +2836,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -2973,24 +2981,32 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: value (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string), ds (type: string) TableScan alias: t1 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: value (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string), ds (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3096,11 +3112,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -3127,7 +3143,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int) sort order: + - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) Path -> Alias: @@ -3160,13 +3176,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/ppd2.q.out ql/src/test/results/clientpositive/ppd2.q.out index b24b4db..dd2aa3f 100644 --- ql/src/test/results/clientpositive/ppd2.q.out +++ ql/src/test/results/clientpositive/ppd2.q.out @@ -350,24 +350,24 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -377,11 +377,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -397,16 +397,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_clusterby.q.out ql/src/test/results/clientpositive/ppd_clusterby.q.out index 7b5b786..9150b7c 100644 --- ql/src/test/results/clientpositive/ppd_clusterby.q.out +++ ql/src/test/results/clientpositive/ppd_clusterby.q.out @@ -75,24 +75,24 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -102,14 +102,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = 20) (type: boolean) - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -125,16 +125,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 719 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -232,24 +232,24 @@ STAGE PLANS: alias: y Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 20)) (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -259,11 +259,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -279,16 +279,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1542 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_gby_join.q.out ql/src/test/results/clientpositive/ppd_gby_join.q.out index 2367ffe..d00c591 100644 --- ql/src/test/results/clientpositive/ppd_gby_join.q.out +++ ql/src/test/results/clientpositive/ppd_gby_join.q.out @@ -31,32 +31,32 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) + predicate: (_col0 is not null and (_col0 < '400')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) @@ -72,7 +72,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -161,22 +161,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -196,7 +196,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE diff --git ql/src/test/results/clientpositive/ppd_join.q.out ql/src/test/results/clientpositive/ppd_join.q.out index 4d241bd..400de16 100644 --- ql/src/test/results/clientpositive/ppd_join.q.out +++ ql/src/test/results/clientpositive/ppd_join.q.out @@ -28,33 +28,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + predicate: ((((key > '2') and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) + predicate: (_col0 is not null and (_col0 < '400')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) @@ -70,7 +70,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -560,7 +560,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + predicate: ((((key > '2') and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -576,7 +576,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) diff --git ql/src/test/results/clientpositive/ppd_join2.q.out ql/src/test/results/clientpositive/ppd_join2.q.out index 4009541..88d4f22 100644 --- ql/src/test/results/clientpositive/ppd_join2.q.out +++ ql/src/test/results/clientpositive/ppd_join2.q.out @@ -33,40 +33,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: NONE + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '302') and (key < '400')) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 is not null and (_col0 < '400')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -74,15 +75,18 @@ STAGE PLANS: Inner Join 0 to 1 condition expressions: 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 660 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -92,43 +96,46 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 6 Data size: 660 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '306') and (sqrt(key) <> 13)) (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (((key <> '306') and value is not null) and (sqrt(key) <> 13)) (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {KEY.reducesinkkey0} + 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1719,34 +1726,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '302') and (key < '400')) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -1754,9 +1762,9 @@ STAGE PLANS: Inner Join 0 to 1 condition expressions: 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1772,43 +1780,43 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '306') and (sqrt(key) <> 13)) (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (((key <> '306') and value is not null) and (sqrt(key) <> 13)) (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {KEY.reducesinkkey0} + 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3636 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3636 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_join3.q.out ql/src/test/results/clientpositive/ppd_join3.q.out index f960684..a07cea4 100644 --- ql/src/test/results/clientpositive/ppd_join3.q.out +++ ql/src/test/results/clientpositive/ppd_join3.q.out @@ -34,58 +34,58 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 is not null and (_col0 < '400')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 500 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 500 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -96,17 +96,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 1100 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 500 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 500 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 500 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1755,49 +1755,49 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1808,17 +1808,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 1982 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 6 Data size: 610 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1043 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 610 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1043 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 610 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1043 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_join4.q.out ql/src/test/results/clientpositive/ppd_join4.q.out index 94154b1..7ee07ae 100644 --- ql/src/test/results/clientpositive/ppd_join4.q.out +++ ql/src/test/results/clientpositive/ppd_join4.q.out @@ -55,7 +55,7 @@ STAGE PLANS: alias: test_tbl Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: ((name = 'c') and (id = 'a')) (type: boolean) + predicate: ((id is not null and (name = 'c')) and (id = 'a')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: id (type: string), name (type: string) @@ -85,7 +85,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (id = 'a') (type: boolean) + predicate: (id is not null and (id = 'a')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: id (type: string) diff --git ql/src/test/results/clientpositive/ppd_join_filter.q.out ql/src/test/results/clientpositive/ppd_join_filter.q.out index 066c821..95855c9 100644 --- ql/src/test/results/clientpositive/ppd_join_filter.q.out +++ ql/src/test/results/clientpositive/ppd_join_filter.q.out @@ -132,23 +132,27 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(key) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -205,15 +209,15 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double) outputColumnNames: _col0, _col2, _col3, _col4 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: (_col2 < 5.0) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 is not null and (_col2 < 5.0)) (type: boolean) + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -239,19 +243,23 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) Path -> Alias: @@ -333,17 +341,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col2} {VALUE._col3} outputColumnNames: _col0, _col7, _col8 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: double), _col8 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -534,23 +542,27 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(key) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -607,15 +619,15 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double) outputColumnNames: _col0, _col2, _col3, _col4 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: (_col2 < 5.0) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -641,19 +653,23 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) Path -> Alias: @@ -735,17 +751,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col2} {VALUE._col3} outputColumnNames: _col0, _col7, _col8 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: double), _col8 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1016,8 +1032,8 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: (_col2 < 5.0) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 is not null and (_col2 < 5.0)) (type: boolean) + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1043,19 +1059,23 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 501 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) Path -> Alias: @@ -1137,17 +1157,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col2} {VALUE._col3} outputColumnNames: _col0, _col7, _col8 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: double), _col8 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1338,23 +1358,27 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(key) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1411,15 +1435,15 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double) outputColumnNames: _col0, _col2, _col3, _col4 - Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: (_col2 < 5.0) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1445,19 +1469,23 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 TableScan GatherStats: false Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) Path -> Alias: @@ -1539,17 +1567,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col2} {VALUE._col3} outputColumnNames: _col0, _col7, _col8 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col7 (type: double), _col8 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/ppd_multi_insert.q.out ql/src/test/results/clientpositive/ppd_multi_insert.q.out index 4264d90..f3cb910 100644 --- ql/src/test/results/clientpositive/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/ppd_multi_insert.q.out @@ -50,20 +50,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -72,17 +78,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -90,14 +96,14 @@ STAGE PLANS: name: default.mi1 Filter Operator predicate: ((_col0 >= 100) and (_col0 < 200)) (type: boolean) - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -105,14 +111,14 @@ STAGE PLANS: name: default.mi2 Filter Operator predicate: ((_col0 >= 200) and (_col0 < 300)) (type: boolean) - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -120,14 +126,14 @@ STAGE PLANS: name: default.mi3 Filter Operator predicate: (_col0 >= 300) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1293,20 +1299,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1315,17 +1327,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1333,14 +1345,14 @@ STAGE PLANS: name: default.mi1 Filter Operator predicate: ((_col0 >= 100) and (_col0 < 200)) (type: boolean) - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1348,14 +1360,14 @@ STAGE PLANS: name: default.mi2 Filter Operator predicate: ((_col0 >= 200) and (_col0 < 300)) (type: boolean) - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 309 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1363,14 +1375,14 @@ STAGE PLANS: name: default.mi3 Filter Operator predicate: (_col0 >= 300) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_random.q.out ql/src/test/results/clientpositive/ppd_random.q.out index 485e1bf..c74e6e9 100644 --- ql/src/test/results/clientpositive/ppd_random.q.out +++ ql/src/test/results/clientpositive/ppd_random.q.out @@ -28,30 +28,39 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '2') and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 1503 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -60,17 +69,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (rand() > 0.5) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 516 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 516 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 516 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -112,30 +121,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '2') and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -144,17 +156,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (rand() > 0.5) (type: boolean) - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 2131 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_repeated_alias.q.out ql/src/test/results/clientpositive/ppd_repeated_alias.q.out index 922c9c8..ccb9c5d 100644 --- ql/src/test/results/clientpositive/ppd_repeated_alias.q.out +++ ql/src/test/results/clientpositive/ppd_repeated_alias.q.out @@ -187,16 +187,19 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: foo (type: int) - sort order: + - Map-reduce partition columns: foo (type: int) + Filter Operator + predicate: foo is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: foo (type: int) + sort order: + + Map-reduce partition columns: foo (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: a Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (bar = 3) (type: boolean) + predicate: (foo is not null and (bar = 3)) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: foo (type: int) diff --git ql/src/test/results/clientpositive/ppd_udf_case.q.out ql/src/test/results/clientpositive/ppd_udf_case.q.out index c5f770f..2b4b562 100644 --- ql/src/test/results/clientpositive/ppd_udf_case.q.out +++ ql/src/test/results/clientpositive/ppd_udf_case.q.out @@ -37,25 +37,25 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string), ds (type: string), hr (type: string) TableScan alias: a Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string), ds (type: string), hr (type: string) Reduce Operator Tree: Join Operator @@ -65,14 +65,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} outputColumnNames: _col0, _col1, _col2, _col3, _col6, _col7, _col8, _col9 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col2 = '2008-04-08') and (_col8 = '2008-04-08')) and CASE (_col0) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 205 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 205 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -87,15 +87,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) sort order: ++++++++ - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 205 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 205 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 205 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -182,25 +182,25 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string), ds (type: string), hr (type: string) TableScan alias: a Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and CASE (key) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string), ds (type: string), hr (type: string) Reduce Operator Tree: Join Operator @@ -210,11 +210,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} outputColumnNames: _col0, _col1, _col2, _col3, _col6, _col7, _col8, _col9 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -229,15 +229,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) sort order: ++++++++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3085 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ppd_union_view.q.out ql/src/test/results/clientpositive/ppd_union_view.q.out index 7b206c5..82406be 100644 --- ql/src/test/results/clientpositive/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/ppd_union_view.q.out @@ -166,24 +166,32 @@ STAGE PLANS: alias: t1_mapping Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: keymap (type: string), ds (type: string) - sort order: ++ - Map-reduce partition columns: keymap (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: key (type: string) + Reduce Output Operator + key expressions: keymap (type: string), ds (type: string) + sort order: ++ + Map-reduce partition columns: keymap (type: string), ds (type: string) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) TableScan alias: t1_old Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: keymap (type: string), ds (type: string) - sort order: ++ - Map-reduce partition columns: keymap (type: string), ds (type: string) + Filter Operator + isSamplingPred: false + predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: value (type: string) + Reduce Output Operator + key expressions: keymap (type: string), ds (type: string) + sort order: ++ + Map-reduce partition columns: keymap (type: string), ds (type: string) + Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/ppd_vc.q.out ql/src/test/results/clientpositive/ppd_vc.q.out index 5657226..928986f 100644 --- ql/src/test/results/clientpositive/ppd_vc.q.out +++ ql/src/test/results/clientpositive/ppd_vc.q.out @@ -412,29 +412,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + tag: 0 TableScan alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: ((BLOCK__OFFSET__INSIDE__FILE < 100) and (BLOCK__OFFSET__INSIDE__FILE < 50)) (type: boolean) - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + predicate: (((BLOCK__OFFSET__INSIDE__FILE < 100) and (BLOCK__OFFSET__INSIDE__FILE < 50)) and key is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 1202 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), ds (type: string), hr (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1202 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 2404 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1202 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) Path -> Alias: @@ -683,11 +687,11 @@ STAGE PLANS: 0 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} outputColumnNames: _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -714,7 +718,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string), _col4 (type: bigint) sort order: +++ - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string) Path -> Alias: @@ -747,13 +751,13 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/quotedid_skew.q.out ql/src/test/results/clientpositive/quotedid_skew.q.out index 68db89a..b112f7a 100644 --- ql/src/test/results/clientpositive/quotedid_skew.q.out +++ ql/src/test/results/clientpositive/quotedid_skew.q.out @@ -58,7 +58,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (!@#$%^&*()_q = '2')) (type: boolean) + predicate: (!@#$%^&*()_q is not null and (not (!@#$%^&*()_q = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: !@#$%^&*()_q (type: string) @@ -70,7 +70,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (!@#$%^&*()_q = '2')) (type: boolean) + predicate: (!@#$%^&*()_q is not null and (not (!@#$%^&*()_q = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: !@#$%^&*()_q (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (!@#$%^&*()_q = '2') (type: boolean) + predicate: (!@#$%^&*()_q is not null and (!@#$%^&*()_q = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: !@#$%^&*()_q (type: string) @@ -147,7 +147,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (!@#$%^&*()_q = '2') (type: boolean) + predicate: (!@#$%^&*()_q is not null and (!@#$%^&*()_q = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: !@#$%^&*()_q (type: string) diff --git ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out index 238d1c8..df394dc 100644 --- ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out +++ ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out @@ -15,21 +15,24 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -48,13 +51,16 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -70,11 +76,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out index f6d540e..96a7ba9 100644 --- ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out +++ ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out @@ -245,20 +245,26 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -267,17 +273,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -293,7 +299,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: double) Reduce Operator Tree: Group By Operator @@ -301,14 +307,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -338,20 +344,26 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -360,11 +372,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -379,15 +391,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2732,20 +2744,26 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -2754,11 +2772,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2774,21 +2792,21 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(KEY._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2818,20 +2836,26 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -2840,11 +2864,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2859,15 +2883,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/regex_col.q.out ql/src/test/results/clientpositive/regex_col.q.out index e1f430a..c1674ef 100644 --- ql/src/test/results/clientpositive/regex_col.q.out +++ ql/src/test/results/clientpositive/regex_col.q.out @@ -110,21 +110,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string), hr (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: ds (type: string), hr (type: string) TableScan alias: a Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string), hr (type: string) + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: ds (type: string), hr (type: string) Reduce Operator Tree: Join Operator condition map: @@ -133,14 +139,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col2, _col3, _col8, _col9 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -175,7 +181,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 103) (type: boolean) + predicate: key is not null (type: boolean) Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), hr (type: string), ds (type: string) @@ -186,13 +192,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 232 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 103) (type: boolean) - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + predicate: ((key = 103) and key is not null) (type: boolean) + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), hr (type: string), ds (type: string) sort order: +++ Map-reduce partition columns: key (type: string), hr (type: string), ds (type: string) - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/sample8.q.out ql/src/test/results/clientpositive/sample8.q.out index 7c37248..1aedf4c 100644 --- ql/src/test/results/clientpositive/sample8.q.out +++ ql/src/test/results/clientpositive/sample8.q.out @@ -124,28 +124,28 @@ STAGE PLANS: Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator - isSamplingPred: true - predicate: ((((hash(key) & 2147483647) % 1) = 0) and (((hash(key) & 2147483647) % 10) = 0)) (type: boolean) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + isSamplingPred: false + predicate: (((((hash(key) & 2147483647) % 10) = 0) and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE tag: 1 TableScan alias: s Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator - isSamplingPred: true - predicate: ((((hash(key) & 2147483647) % 10) = 0) and (((hash(key) & 2147483647) % 1) = 0)) (type: boolean) - Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + isSamplingPred: false + predicate: (((((hash(key) & 2147483647) % 1) = 0) and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: ds (type: string), hr (type: string) Path -> Alias: @@ -349,7 +349,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} {VALUE._col1} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} outputColumnNames: _col0, _col1, _col2, _col3, _col6, _col7 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: ((((((_col6 = _col0) and (_col7 = _col1)) and (_col2 = '2008-04-08')) and (_col3 = '11')) and (_col2 = '2008-04-08')) and (_col3 = '11')) (type: boolean) diff --git ql/src/test/results/clientpositive/select_transform_hint.q.out ql/src/test/results/clientpositive/select_transform_hint.q.out index 5a3a93a..c886111 100644 --- ql/src/test/results/clientpositive/select_transform_hint.q.out +++ ql/src/test/results/clientpositive/select_transform_hint.q.out @@ -27,13 +27,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-1 Map Reduce @@ -41,35 +44,38 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work @@ -1144,20 +1150,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1166,21 +1178,21 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Transform Operator command: cat output info: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/semijoin.q.out ql/src/test/results/clientpositive/semijoin.q.out index b32b336..76dcfe4 100644 --- ql/src/test/results/clientpositive/semijoin.q.out +++ ql/src/test/results/clientpositive/semijoin.q.out @@ -120,29 +120,35 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -151,11 +157,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -170,15 +176,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -222,29 +228,35 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -253,11 +265,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -272,15 +284,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -326,29 +338,35 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 + Select Operator + expressions: key (type: int) + outputColumnNames: key Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -357,11 +375,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -376,15 +394,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -423,31 +441,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 15) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 15) and key is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: int), key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -456,11 +477,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -475,15 +496,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -533,31 +554,34 @@ STAGE PLANS: alias: b Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value < 'val_10') (type: boolean) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + predicate: ((value < 'val_10') and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: int), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -566,11 +590,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -585,15 +609,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -634,32 +658,35 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: t3 Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 5) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 5) and key is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -668,11 +695,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -687,15 +714,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -737,7 +764,7 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 5) and (value <= 'val_20')) (type: boolean) + predicate: (((key > 5) and (value <= 'val_20')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -756,12 +783,15 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -770,11 +800,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -789,15 +819,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -836,31 +866,34 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 2) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 2) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -869,11 +902,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -888,15 +921,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -944,22 +977,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) Stage: Stage-1 Map Reduce @@ -967,35 +1003,38 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Left Semi Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1052,29 +1091,35 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (2 * _col0) (type: int) - sort order: + - Map-reduce partition columns: (2 * _col0) (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (2 * key) is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (2 * _col0) (type: int) + sort order: + + Map-reduce partition columns: (2 * _col0) (type: int) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1083,11 +1128,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1102,15 +1147,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1152,38 +1197,47 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: c Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1194,11 +1248,11 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1213,16 +1267,16 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: int), _col3 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1276,28 +1330,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: string) - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: string) + Statistics: Num rows: 6 Data size: 44 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1306,11 +1366,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1325,15 +1385,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1393,46 +1453,52 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) c TableScan alias: c Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} - 1 - 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) Stage: Stage-1 Map Reduce @@ -1440,38 +1506,41 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Left Semi Join 0 to 1 - Left Semi Join 0 to 2 - condition expressions: - 0 {key} - 1 - 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Left Semi Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2520,31 +2589,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (key > 100)) (type: boolean) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: int) Reduce Operator Tree: Join Operator @@ -2554,14 +2626,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/skewjoin.q.out ql/src/test/results/clientpositive/skewjoin.q.out index 1a1908a..596855d 100644 --- ql/src/test/results/clientpositive/skewjoin.q.out +++ ql/src/test/results/clientpositive/skewjoin.q.out @@ -88,20 +88,26 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -111,14 +117,14 @@ STAGE PLANS: 1 {VALUE._col0} handleSkewJoin: true outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -229,39 +235,51 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -274,14 +292,14 @@ STAGE PLANS: 2 {KEY.reducesinkkey0} {VALUE._col0} 3 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -337,39 +355,51 @@ STAGE PLANS: TableScan alias: d Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -382,14 +412,14 @@ STAGE PLANS: 2 {KEY.reducesinkkey0} {VALUE._col0} 3 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string), _col12 (type: string), _col13 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 99 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -438,21 +468,27 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -461,11 +497,11 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col4)) mode: hash @@ -550,28 +586,34 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -581,11 +623,11 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} handleSkewJoin: true outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2)), sum(hash(_col3)) mode: hash @@ -728,28 +770,34 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and UDFToDouble(substring(value, 5)) is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: (key is not null and (substring(value, 5) + 1) is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -759,11 +807,11 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} handleSkewJoin: true outputColumnNames: _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2)), sum(hash(_col3)) mode: hash @@ -917,48 +965,48 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 80) and (key < 100)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -970,11 +1018,11 @@ STAGE PLANS: 2 handleSkewJoin: true outputColumnNames: _col0, _col3 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 13 Data size: 1322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 33 Data size: 6613 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col3)) mode: hash diff --git ql/src/test/results/clientpositive/skewjoin_noskew.q.out ql/src/test/results/clientpositive/skewjoin_noskew.q.out index a0cfe6c..44e920e 100644 --- ql/src/test/results/clientpositive/skewjoin_noskew.q.out +++ ql/src/test/results/clientpositive/skewjoin_noskew.q.out @@ -21,20 +21,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -44,11 +50,11 @@ STAGE PLANS: 1 handleSkewJoin: true outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -109,19 +115,19 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 30 - Statistics: Num rows: 30 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 3090 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 30 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 3090 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out index 34245f3..7678cb9 100644 --- ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out +++ ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out @@ -71,7 +71,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -83,7 +83,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -119,7 +119,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -131,7 +131,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -350,7 +350,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -362,7 +362,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -409,7 +409,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -421,7 +421,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out index 7a1c0ed..bb9af2f 100644 --- ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out +++ ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out @@ -82,7 +82,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -94,7 +94,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -106,7 +106,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -144,7 +144,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -156,7 +156,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -168,7 +168,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt1.q.out ql/src/test/results/clientpositive/skewjoinopt1.q.out index ea1194f..eeeca5a 100644 --- ql/src/test/results/clientpositive/skewjoinopt1.q.out +++ ql/src/test/results/clientpositive/skewjoinopt1.q.out @@ -58,7 +58,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -70,7 +70,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -147,7 +147,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -390,7 +390,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -401,7 +401,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -483,7 +483,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -494,7 +494,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt10.q.out ql/src/test/results/clientpositive/skewjoinopt10.q.out index 1196da7..99789f9 100644 --- ql/src/test/results/clientpositive/skewjoinopt10.q.out +++ ql/src/test/results/clientpositive/skewjoinopt10.q.out @@ -60,7 +60,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '8')) (type: boolean) + predicate: (key is not null and (not (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -71,13 +71,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not (key = '8')) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (not (key = '8'))) (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: array) Reduce Operator Tree: Join Operator @@ -87,11 +87,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -104,25 +104,25 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -130,45 +130,45 @@ STAGE PLANS: Select Operator expressions: _col1 (type: array) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Forward - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -176,20 +176,20 @@ STAGE PLANS: Select Operator expressions: _col1 (type: array) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE UDTF Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE function name: explode Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -202,7 +202,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '8') (type: boolean) + predicate: (key is not null and (key = '8')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -213,13 +213,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = '8') (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = '8')) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: array) Reduce Operator Tree: Join Operator @@ -229,11 +229,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/skewjoinopt11.q.out ql/src/test/results/clientpositive/skewjoinopt11.q.out index 432a4fd..f1457de 100644 --- ql/src/test/results/clientpositive/skewjoinopt11.q.out +++ ql/src/test/results/clientpositive/skewjoinopt11.q.out @@ -74,7 +74,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -86,7 +86,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -181,7 +181,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -193,7 +193,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -228,7 +228,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -240,7 +240,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -303,7 +303,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -315,7 +315,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt12.q.out ql/src/test/results/clientpositive/skewjoinopt12.q.out index b8eed90..f9cfd89 100644 --- ql/src/test/results/clientpositive/skewjoinopt12.q.out +++ ql/src/test/results/clientpositive/skewjoinopt12.q.out @@ -60,7 +60,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13')))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13'))))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -71,7 +71,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13')))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13'))))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13'))) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -146,7 +146,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13'))) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') and (val = '12')) or ((key = '8') and (val = '18'))) or ((key = '3') and (val = '13')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt13.q.out ql/src/test/results/clientpositive/skewjoinopt13.q.out index 937ca88..1bcc1bf 100644 --- ql/src/test/results/clientpositive/skewjoinopt13.q.out +++ ql/src/test/results/clientpositive/skewjoinopt13.q.out @@ -83,21 +83,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: (key is not null and val is not null) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -106,7 +112,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -122,17 +128,20 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: val (type: string) - sort order: + - Map-reduce partition columns: val (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: val is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val (type: string) + sort order: + + Map-reduce partition columns: val (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -141,14 +150,14 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col3} {VALUE._col4} 1 {VALUE._col0} {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/skewjoinopt14.q.out ql/src/test/results/clientpositive/skewjoinopt14.q.out index fd774e8..e9ae094 100644 --- ql/src/test/results/clientpositive/skewjoinopt14.q.out +++ ql/src/test/results/clientpositive/skewjoinopt14.q.out @@ -89,7 +89,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: ((key is not null and val is not null) and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -101,7 +101,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -155,12 +155,15 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: val (type: string) - sort order: + - Map-reduce partition columns: val (type: string) - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string) + Filter Operator + predicate: val is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val (type: string) + sort order: + + Map-reduce partition columns: val (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -189,7 +192,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -201,7 +204,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: ((key is not null and val is not null) and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt15.q.out ql/src/test/results/clientpositive/skewjoinopt15.q.out index da3dec5..52f1db3 100644 --- ql/src/test/results/clientpositive/skewjoinopt15.q.out +++ ql/src/test/results/clientpositive/skewjoinopt15.q.out @@ -96,25 +96,25 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (key is not null and (not ((key = 2) or (key = 3)))) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) TableScan alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (key is not null and (not ((key = 2) or (key = 3)))) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) Reduce Operator Tree: Join Operator @@ -124,11 +124,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -141,26 +141,26 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -173,25 +173,25 @@ STAGE PLANS: alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and ((key = 2) or (key = 3))) (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) TableScan alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and ((key = 2) or (key = 3))) (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) Reduce Operator Tree: Join Operator @@ -201,11 +201,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -428,24 +428,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (key is not null and (not ((key = 2) or (key = 3)))) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TableScan alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (key is not null and (not ((key = 2) or (key = 3)))) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -453,9 +453,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -468,10 +468,10 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash @@ -483,10 +483,10 @@ STAGE PLANS: value expressions: _col0 (type: bigint) TableScan Union - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash @@ -521,24 +521,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and ((key = 2) or (key = 3))) (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and ((key = 2) or (key = 3))) (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -546,9 +546,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/skewjoinopt16.q.out ql/src/test/results/clientpositive/skewjoinopt16.q.out index 402f2ed..c6397e2 100644 --- ql/src/test/results/clientpositive/skewjoinopt16.q.out +++ ql/src/test/results/clientpositive/skewjoinopt16.q.out @@ -60,7 +60,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') and (val = '12')) or (key = '3'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not (((key = '2') and (val = '12')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -71,7 +71,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') and (val = '12')) or (key = '3'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not (((key = '2') and (val = '12')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') and (val = '12')) or (key = '3')) (type: boolean) + predicate: ((key is not null and val is not null) and (((key = '2') and (val = '12')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -146,7 +146,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') and (val = '12')) or (key = '3')) (type: boolean) + predicate: ((key is not null and val is not null) and (((key = '2') and (val = '12')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt17.q.out ql/src/test/results/clientpositive/skewjoinopt17.q.out index fdae32d..7d47030 100644 --- ql/src/test/results/clientpositive/skewjoinopt17.q.out +++ ql/src/test/results/clientpositive/skewjoinopt17.q.out @@ -64,7 +64,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -76,7 +76,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -141,7 +141,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -153,7 +153,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -283,7 +283,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') and (val = '12')) or (key = '2'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not (((key = '2') and (val = '12')) or (key = '2')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -294,7 +294,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') and (val = '12')) or (key = '2'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not (((key = '2') and (val = '12')) or (key = '2')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -358,7 +358,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') and (val = '12')) or (key = '2')) (type: boolean) + predicate: ((key is not null and val is not null) and (((key = '2') and (val = '12')) or (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -369,7 +369,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') and (val = '12')) or (key = '2')) (type: boolean) + predicate: ((key is not null and val is not null) and (((key = '2') and (val = '12')) or (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt18.q.out ql/src/test/results/clientpositive/skewjoinopt18.q.out index 2c89f44..f1d2131 100644 --- ql/src/test/results/clientpositive/skewjoinopt18.q.out +++ ql/src/test/results/clientpositive/skewjoinopt18.q.out @@ -82,21 +82,27 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: key (type: string), val (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int), val (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -105,14 +111,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/skewjoinopt19.q.out ql/src/test/results/clientpositive/skewjoinopt19.q.out index b0be394..257e108 100644 --- ql/src/test/results/clientpositive/skewjoinopt19.q.out +++ ql/src/test/results/clientpositive/skewjoinopt19.q.out @@ -62,7 +62,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -74,7 +74,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -139,7 +139,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -151,7 +151,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt2.q.out ql/src/test/results/clientpositive/skewjoinopt2.q.out index 2ea6eb8..fb06a4b 100644 --- ql/src/test/results/clientpositive/skewjoinopt2.q.out +++ ql/src/test/results/clientpositive/skewjoinopt2.q.out @@ -64,7 +64,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -75,7 +75,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -139,7 +139,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -150,7 +150,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -384,7 +384,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -395,7 +395,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((key is not null and val is not null) and (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -487,7 +487,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) @@ -498,7 +498,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: ((key is not null and val is not null) and ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string), val (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt20.q.out ql/src/test/results/clientpositive/skewjoinopt20.q.out index 1a1347a..31ace72 100644 --- ql/src/test/results/clientpositive/skewjoinopt20.q.out +++ ql/src/test/results/clientpositive/skewjoinopt20.q.out @@ -62,7 +62,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -74,7 +74,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -139,7 +139,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -151,7 +151,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt3.q.out ql/src/test/results/clientpositive/skewjoinopt3.q.out index 88113c2..842f8c1 100644 --- ql/src/test/results/clientpositive/skewjoinopt3.q.out +++ ql/src/test/results/clientpositive/skewjoinopt3.q.out @@ -62,7 +62,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -74,7 +74,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -139,7 +139,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -151,7 +151,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt4.q.out ql/src/test/results/clientpositive/skewjoinopt4.q.out index 97aa289..1dc65b5 100644 --- ql/src/test/results/clientpositive/skewjoinopt4.q.out +++ ql/src/test/results/clientpositive/skewjoinopt4.q.out @@ -58,7 +58,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -70,7 +70,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -147,7 +147,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -221,7 +221,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -233,7 +233,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -298,7 +298,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -310,7 +310,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key = '2') (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt5.q.out ql/src/test/results/clientpositive/skewjoinopt5.q.out index 26a73bf..cc1ca52 100644 --- ql/src/test/results/clientpositive/skewjoinopt5.q.out +++ ql/src/test/results/clientpositive/skewjoinopt5.q.out @@ -60,7 +60,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -72,7 +72,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: (key is not null and (not ((key = '2') or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -137,7 +137,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -149,7 +149,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (key is not null and ((key = '2') or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt6.q.out ql/src/test/results/clientpositive/skewjoinopt6.q.out index 3437056..f7dfd7f 100644 --- ql/src/test/results/clientpositive/skewjoinopt6.q.out +++ ql/src/test/results/clientpositive/skewjoinopt6.q.out @@ -62,7 +62,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -74,7 +74,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -139,7 +139,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -151,7 +151,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt7.q.out ql/src/test/results/clientpositive/skewjoinopt7.q.out index f7663f1..12cc3fb 100644 --- ql/src/test/results/clientpositive/skewjoinopt7.q.out +++ ql/src/test/results/clientpositive/skewjoinopt7.q.out @@ -77,7 +77,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -89,7 +89,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -101,7 +101,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + predicate: (key is not null and (not (((key = '2') or (key = '8')) or (key = '3')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -168,7 +168,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -180,7 +180,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -192,7 +192,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + predicate: (key is not null and (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt8.q.out ql/src/test/results/clientpositive/skewjoinopt8.q.out index 7560bb3..04c3975 100644 --- ql/src/test/results/clientpositive/skewjoinopt8.q.out +++ ql/src/test/results/clientpositive/skewjoinopt8.q.out @@ -75,7 +75,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '3') or (key = '8'))) (type: boolean) + predicate: (key is not null and (not ((key = '3') or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -87,7 +87,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '3') or (key = '8'))) (type: boolean) + predicate: (key is not null and (not ((key = '3') or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -99,7 +99,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (not ((key = '3') or (key = '8'))) (type: boolean) + predicate: (key is not null and (not ((key = '3') or (key = '8')))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -166,7 +166,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '3') or (key = '8')) (type: boolean) + predicate: (key is not null and ((key = '3') or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -178,7 +178,7 @@ STAGE PLANS: alias: c Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '3') or (key = '8')) (type: boolean) + predicate: (key is not null and ((key = '3') or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -190,7 +190,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((key = '3') or (key = '8')) (type: boolean) + predicate: (key is not null and ((key = '3') or (key = '8'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) diff --git ql/src/test/results/clientpositive/skewjoinopt9.q.out ql/src/test/results/clientpositive/skewjoinopt9.q.out index 59b6e94..cd99523 100644 --- ql/src/test/results/clientpositive/skewjoinopt9.q.out +++ ql/src/test/results/clientpositive/skewjoinopt9.q.out @@ -65,42 +65,51 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Union - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Union - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -109,14 +118,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 66 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -194,22 +203,25 @@ STAGE PLANS: TableScan alias: t1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count(1) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -241,12 +253,15 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - value expressions: val (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/smb_mapjoin_1.q.out ql/src/test/results/clientpositive/smb_mapjoin_1.q.out index 9e42b8c..bcc51b7 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_1.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_1.q.out @@ -60,25 +60,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -290,25 +293,28 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_10.q.out ql/src/test/results/clientpositive/smb_mapjoin_10.q.out index 650e14f..3fb7fd1 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_10.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_10.q.out @@ -82,25 +82,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 3 Data size: 414 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {userid} {pageid} {postid} {type} {ds} - 1 {userid} {pageid} {postid} {type} {ds} - keys: - 0 userid (type: int), pageid (type: int), postid (type: int), type (type: string) - 1 userid (type: int), pageid (type: int), postid (type: int), type (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col8, _col9, _col10, _col11 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (((userid is not null and pageid is not null) and postid is not null) and type is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 138 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {userid} {pageid} {postid} {type} {ds} + 1 {userid} {pageid} {postid} {type} {ds} + keys: + 0 userid (type: int), pageid (type: int), postid (type: int), type (type: string) + 1 userid (type: int), pageid (type: int), postid (type: int), type (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col7, _col8, _col9, _col10, _col11 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/smb_mapjoin_11.q.out index bbc41e4..d59b801 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_11.q.out @@ -129,49 +129,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col6 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col6 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=1/ + NumFilesPerFileSink: 1 + Static Partition Specification: ds=1/ #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 16 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.test_table3 - partition_columns ds - partition_columns.types string - serialization.ddl struct test_table3 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/smb_mapjoin_12.q.out index 4702e40..ab3f93f 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_12.q.out @@ -141,50 +141,54 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col6 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=1/ -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 16 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.test_table3 - partition_columns ds - partition_columns.types string - serialization.ddl struct test_table3 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col6 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=1/ +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -400,50 +404,54 @@ STAGE PLANS: alias: a Statistics: Num rows: 3084 Data size: 32904 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), concat(_col1, _col6) (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2/ -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 16 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.test_table3 - partition_columns ds - partition_columns.types string - serialization.ddl struct test_table3 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 1542 Data size: 16452 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col6) (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 16 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.test_table3 + partition_columns ds + partition_columns.types string + serialization.ddl struct test_table3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/smb_mapjoin_13.q.out ql/src/test/results/clientpositive/smb_mapjoin_13.q.out index bcee46d..e033da1 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_13.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_13.q.out @@ -128,25 +128,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} {key} - keys: - 0 key (type: int) - 1 value (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - tag: -1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} {key} + keys: + 0 key (type: int) + 1 value (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + tag: -1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -326,14 +330,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(value) (type: double) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(value) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(value) (type: double) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -342,28 +350,32 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(value) (type: double) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + Filter Operator + isSamplingPred: false + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(value) (type: double) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) Local Work: Map Reduce Local Work Path -> Alias: @@ -424,7 +436,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/smb_mapjoin_14.q.out ql/src/test/results/clientpositive/smb_mapjoin_14.q.out index 98aeee2..5590f92 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_14.q.out @@ -59,23 +59,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -144,29 +147,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -268,29 +274,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0 - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -392,12 +401,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -494,23 +503,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -615,7 +627,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) + predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -732,12 +744,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 8) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -834,13 +846,16 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Stage: Stage-1 Map Reduce @@ -852,27 +867,30 @@ STAGE PLANS: expressions: (key + 1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -943,23 +961,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1022,12 +1043,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1117,12 +1138,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 6) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1230,23 +1251,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 _col0 (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/smb_mapjoin_15.q.out ql/src/test/results/clientpositive/smb_mapjoin_15.q.out index 3368c7d..2a4bc28 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_15.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_15.q.out @@ -102,25 +102,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - tag: -1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + tag: -1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -354,25 +358,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {key} {key2} {value} - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - tag: -1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Filter Operator + isSamplingPred: false + predicate: (key is not null and key2 is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1804 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {key} {key2} {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + tag: -1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -556,25 +564,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {key} {key2} {value} - keys: - 0 key2 (type: int), key (type: int) - 1 key2 (type: int), key (type: int) - outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 - Position of Big Table: 0 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - tag: -1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Filter Operator + isSamplingPred: false + predicate: (key2 is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1804 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {key} {key2} {value} + keys: + 0 key2 (type: int), key (type: int) + 1 key2 (type: int), key (type: int) + outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + tag: -1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -764,14 +776,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {key2} {value} - 1 {key2} - keys: - 0 key (type: int), value (type: string) - 1 key (type: int), value (type: string) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1804 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {key2} + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {000000_0=[000000_0], 000001_0=[000001_0], 000002_0=[000002_0], 000003_0=[000003_0], 000004_0=[000004_0], 000005_0=[000005_0], 000006_0=[000006_0], 000007_0=[000007_0], 000008_0=[000008_0], 000009_0=[000009_0], 000010_0=[000010_0], 000011_0=[000011_0], 000012_0=[000012_0], 000013_0=[000013_0], 000014_0=[000014_0], 000015_0=[000015_0]} @@ -787,29 +803,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {key2} {value} - 1 {key} {key2} {value} - keys: - 0 key (type: int), value (type: string) - 1 key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Filter Operator + isSamplingPred: false + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1804 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {key2} {value} + 1 {key} {key2} {value} + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col7 + Position of Big Table: 0 + Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Local Work: Map Reduce Local Work Path -> Alias: @@ -870,7 +890,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1984 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/smb_mapjoin_16.q.out ql/src/test/results/clientpositive/smb_mapjoin_16.q.out index 462d6c1..a979a64 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_16.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_16.q.out @@ -51,23 +51,26 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/smb_mapjoin_17.q.out ql/src/test/results/clientpositive/smb_mapjoin_17.q.out index 68eaff0..9ead57a 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_17.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_17.q.out @@ -183,38 +183,41 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - Inner Join 0 to 4 - Inner Join 0 to 5 - Inner Join 0 to 6 - condition expressions: - 0 - 1 - 2 - 3 - 4 - 5 - 6 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - 3 key (type: int) - 4 key (type: int) - 5 key (type: int) - 6 key (type: int) - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + Inner Join 0 to 6 + condition expressions: + 0 + 1 + 2 + 3 + 4 + 5 + 6 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + 3 key (type: int) + 4 key (type: int) + 5 key (type: int) + 6 key (type: int) + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/smb_mapjoin_2.q.out ql/src/test/results/clientpositive/smb_mapjoin_2.q.out index f88d12e..a5f7e66 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_2.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_2.q.out @@ -60,25 +60,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 111 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -294,25 +297,28 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_25.q.out ql/src/test/results/clientpositive/smb_mapjoin_25.q.out index 25493c8..43daa7a 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_25.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_25.q.out @@ -63,24 +63,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -89,11 +89,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -109,13 +109,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -124,17 +124,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 = 5) (type: boolean) - Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -147,24 +147,24 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE TableScan alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -173,11 +173,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -219,8 +219,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -242,8 +242,8 @@ STAGE PLANS: alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key = 5)) (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/smb_mapjoin_3.q.out ql/src/test/results/clientpositive/smb_mapjoin_3.q.out index e5de546..379210f 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_3.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_3.q.out @@ -60,25 +60,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 111 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -292,25 +295,28 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_4.q.out ql/src/test/results/clientpositive/smb_mapjoin_4.q.out index 9b8358c..67d9f32 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_4.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_4.q.out @@ -60,28 +60,31 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string), _col8 (type: int), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 111 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string), _col8 (type: int), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_5.q.out ql/src/test/results/clientpositive/smb_mapjoin_5.q.out index ef3f63d..6dbf754 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_5.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_5.q.out @@ -60,28 +60,31 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} {value} - 1 {key} {value} - 2 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string), _col8 (type: int), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string), _col8 (type: int), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/smb_mapjoin_6.q.out index aeccb72..0eee3ed 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_6.q.out @@ -75,26 +75,29 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.smb_join_results + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results Stage: Stage-7 Conditional Operator @@ -1257,26 +1260,29 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.smb_join_results + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.smb_join_results Stage: Stage-8 Conditional Operator @@ -2456,8 +2462,8 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key > 1000)) (type: boolean) + Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -2573,8 +2579,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key > 1000)) (type: boolean) + Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -2682,8 +2688,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key > 1000)) (type: boolean) + Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out index e399791..7009535 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out @@ -64,8 +64,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out index a656761..f8cb20e 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out @@ -72,8 +72,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean) + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out index e944472..add2bc5 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out @@ -72,8 +72,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean) + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out index b408d15..6daf6bc 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out @@ -76,8 +76,8 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean) + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -93,8 +93,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean) + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -104,9 +104,9 @@ STAGE PLANS: keys: 0 key (type: string), value (type: string) 1 key (type: string), value (type: string) - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out index d5e982d..5ff45d9 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out @@ -121,25 +121,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Select Operator - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: _col0 (type: bigint) + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Select Operator + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out index 9d1f1d9..6fdf738 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out @@ -173,14 +173,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {part=1/000000_0=[part=1/000000_0], part=1/000001_0=[part=1/000001_0]} @@ -196,30 +200,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out index d1024dd..08b9d1e 100644 --- ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out +++ ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out @@ -254,14 +254,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Stage: Stage-1 Map Reduce @@ -270,29 +274,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/stats11.q.out ql/src/test/results/clientpositive/stats11.q.out index fdba80b..c5531c5 100644 --- ql/src/test/results/clientpositive/stats11.q.out +++ ql/src/test/results/clientpositive/stats11.q.out @@ -435,14 +435,18 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} @@ -458,49 +462,53 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 0 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -928,14 +936,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false - HashTable Sink Operator - condition expressions: - 0 {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - Position of Big Table: 1 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 Bucket Mapjoin Context: Alias Bucket Base File Name Mapping: a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} @@ -951,54 +963,58 @@ STAGE PLANS: alias: b Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - BucketMapJoin: true - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6393 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Position of Big Table: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + BucketMapJoin: true + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 30 Data size: 3253 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: diff --git ql/src/test/results/clientpositive/subquery_exists.q.out ql/src/test/results/clientpositive/subquery_exists.q.out index 180a163..9313da9 100644 --- ql/src/test/results/clientpositive/subquery_exists.q.out +++ ql/src/test/results/clientpositive/subquery_exists.q.out @@ -29,31 +29,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string), key (type: string) - sort order: ++ - Map-reduce partition columns: value (type: string), key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string), key (type: string) + sort order: ++ + Map-reduce partition columns: value (type: string), key (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((value > 'val_9') and value is not null) and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string), _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -62,17 +65,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/subquery_exists_having.q.out ql/src/test/results/clientpositive/subquery_exists_having.q.out index 56986b7..2194143 100644 --- ql/src/test/results/clientpositive/subquery_exists_having.q.out +++ ql/src/test/results/clientpositive/subquery_exists_having.q.out @@ -32,29 +32,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -70,28 +73,28 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -100,17 +103,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/subquery_in.q.out ql/src/test/results/clientpositive/subquery_in.q.out index 2069101..562d6af 100644 --- ql/src/test/results/clientpositive/subquery_in.q.out +++ ql/src/test/results/clientpositive/subquery_in.q.out @@ -120,31 +120,34 @@ STAGE PLANS: alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -153,17 +156,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -229,31 +232,34 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -262,17 +268,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -392,21 +398,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: double) - outputColumnNames: _col0 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: double) - mode: hash + Select Operator + expressions: _col0 (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-3 Map Reduce @@ -420,12 +429,15 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(p_size) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(p_size) (type: double) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_size (type: int) + Filter Operator + predicate: UDFToDouble(p_size) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(p_size) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(p_size) (type: double) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string), p_size (type: int) Reduce Operator Tree: Join Operator condition map: @@ -434,17 +446,17 @@ STAGE PLANS: 0 {VALUE._col1} {VALUE._col5} 1 outputColumnNames: _col1, _col5 - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1744 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -521,18 +533,18 @@ STAGE PLANS: PTF Operator Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_wcol0 <= 2) (type: boolean) - Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE + predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -548,7 +560,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1057 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) Reduce Operator Tree: Group By Operator @@ -556,22 +568,25 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-1 Map Reduce @@ -579,18 +594,21 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_size (type: int), p_mfgr (type: string) - sort order: ++ - Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string) + Filter Operator + predicate: (p_size is not null and p_mfgr is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 846 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_size (type: int), p_mfgr (type: string) + sort order: ++ + Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) + Statistics: Num rows: 4 Data size: 846 Basic stats: COMPLETE Column stats: NONE + value expressions: p_name (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -599,17 +617,17 @@ STAGE PLANS: 0 {VALUE._col1} {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 16 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 930 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 465 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 465 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 465 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -639,12 +657,12 @@ from part b where b.p_size in POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### +Manufacturer#3 almond antique misty red olive 1 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#2 almond aquamarine midnight light salmon 2 -Manufacturer#3 almond antique misty red olive 1 -Manufacturer#4 almond aquamarine yellow dodger mint 7 Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#4 almond aquamarine yellow dodger mint 7 PREHOOK: query: -- distinct, corr explain select * @@ -678,37 +696,37 @@ STAGE PLANS: alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -722,17 +740,20 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -741,17 +762,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -810,12 +831,12 @@ where part.p_size in POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### +Manufacturer#3 almond antique misty red olive 1 Manufacturer#1 almond antique burnished rose metallic 2 -Manufacturer#1 almond antique burnished rose metallic 2 +Manufacturer#5 almond antique sky peru orange 2 Manufacturer#2 almond aquamarine midnight light salmon 2 -Manufacturer#3 almond antique misty red olive 1 +Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#4 almond aquamarine yellow dodger mint 7 -Manufacturer#5 almond antique sky peru orange 2 PREHOOK: query: -- non agg, non corr, with join in Parent Query explain select p.p_partkey, li.l_suppkey @@ -843,30 +864,33 @@ STAGE PLANS: TableScan alias: lineitem Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: l_partkey (type: int) - outputColumnNames: l_partkey - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: l_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: l_partkey (type: int) + outputColumnNames: l_partkey + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: l_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -881,20 +905,20 @@ STAGE PLANS: alias: li Statistics: Num rows: 756 Data size: 12099 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (l_linenumber = 1) (type: boolean) - Statistics: Num rows: 378 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + predicate: ((l_partkey is not null and l_orderkey is not null) and (l_linenumber = 1)) (type: boolean) + Statistics: Num rows: 94 Data size: 1504 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: l_partkey (type: int) sort order: + Map-reduce partition columns: l_partkey (type: int) - Statistics: Num rows: 378 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 94 Data size: 1504 Basic stats: COMPLETE Column stats: NONE value expressions: l_orderkey (type: int), l_suppkey (type: int) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -903,7 +927,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 1663 Data size: 6653 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -919,28 +943,28 @@ STAGE PLANS: key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1663 Data size: 6653 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: int) TableScan alias: lineitem Statistics: Num rows: 116 Data size: 12099 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 58 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_orderkey (type: int) outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 58 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 58 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -949,14 +973,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col2} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -982,16 +1006,16 @@ where li.l_linenumber = 1 and POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem #### A masked pattern was here #### -108570 8571 -115118 7630 -115209 7721 155190 7706 -2320 9821 -40216 217 4297 1798 +108570 8571 +82704 7721 61336 8855 +2320 9821 +115118 7630 +115209 7721 64128 9141 -82704 7721 +40216 217 PREHOOK: query: -- non agg, corr, with join in Parent Query select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey @@ -1008,5 +1032,5 @@ where li.l_linenumber = 1 and POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem #### A masked pattern was here #### -108570 8571 4297 1798 +108570 8571 diff --git ql/src/test/results/clientpositive/subquery_in_having.q.out ql/src/test/results/clientpositive/subquery_in_having.q.out index 6e1eed0..9e9c15a 100644 --- ql/src/test/results/clientpositive/subquery_in_having.q.out +++ ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -92,21 +92,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - mode: hash + Select Operator + expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -122,27 +125,27 @@ STAGE PLANS: key expressions: _col1 (type: bigint) sort order: + Map-reduce partition columns: _col1 (type: bigint) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} + 0 {VALUE._col0} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -176,13 +179,16 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator @@ -218,37 +224,37 @@ having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1. POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0 3 -119 3 -128 3 -167 3 -187 3 -193 3 -199 3 -208 3 -273 3 -298 3 -311 3 -316 3 -318 3 -327 3 -35 3 +498 3 369 3 384 3 396 3 403 3 409 3 417 3 +5 3 430 3 +70 3 +119 3 +0 3 431 3 438 3 +480 3 +193 3 +199 3 +208 3 +187 3 +273 3 +298 3 454 3 +311 3 +316 3 466 3 -480 3 -498 3 -5 3 -70 3 90 3 +128 3 +318 3 +327 3 +167 3 +35 3 PREHOOK: query: -- non agg, corr explain select key, value, count(*) @@ -276,35 +282,41 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -314,33 +326,33 @@ STAGE PLANS: key expressions: _col2 (type: bigint), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col2 (type: bigint), _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: bigint) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: bigint), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 condition expressions: - 0 {VALUE._col0} {KEY.reducesinkkey1} {VALUE._col1} + 0 {VALUE._col0} {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -353,23 +365,23 @@ STAGE PLANS: alias: s1 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: value, key - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: value (type: string), key (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -377,22 +389,25 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint), _col1 (type: string) - mode: hash + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: bigint), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator @@ -437,29 +452,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_mfgr (type: string), p_size (type: int) - outputColumnNames: p_mfgr, p_size - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: avg(p_size) - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct) + Filter Operator + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_mfgr (type: string), p_size (type: int) + outputColumnNames: p_mfgr, p_size + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(p_size) + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct) Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -475,14 +493,14 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -491,17 +509,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 33 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1744 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 16 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 872 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -513,41 +531,44 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_mfgr (type: string), p_size (type: int) - outputColumnNames: p_mfgr, p_size - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(p_size), min(p_size) - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 30 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int) + Filter Operator + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_mfgr (type: string), p_size (type: int) + outputColumnNames: p_mfgr, p_size + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(p_size), min(p_size) + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int) Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), min(VALUE._col1) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 1586 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 740 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col1 - _col2) < 20) (type: boolean) - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -629,32 +650,35 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '8') (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '8') and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -663,20 +687,20 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -692,7 +716,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -700,13 +724,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-3 Map Reduce @@ -716,8 +743,8 @@ STAGE PLANS: key expressions: _col2 (type: bigint) sort order: + Map-reduce partition columns: _col2 (type: bigint) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: bigint) @@ -729,20 +756,20 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} + 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 3402 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -780,21 +807,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - mode: hash + Select Operator + expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator @@ -873,21 +903,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - mode: hash + Select Operator + expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-7 Conditional Operator @@ -945,8 +978,8 @@ STAGE PLANS: key expressions: _col2 (type: bigint) sort order: + Map-reduce partition columns: _col2 (type: bigint) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: bigint) @@ -958,20 +991,20 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} + 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 3402 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1701 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -989,17 +1022,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '8') (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '8') and key is not null) (type: boolean) + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1002 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} {value} @@ -1014,36 +1047,39 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Left Semi Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 8 Data size: 1653 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -1052,13 +1088,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 4 Data size: 826 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator @@ -1109,21 +1148,24 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _wcol0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Filter Operator + predicate: _wcol0 is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _wcol0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-5 Conditional Operator @@ -1181,13 +1223,13 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: double) Reduce Operator Tree: Join Operator @@ -1197,17 +1239,17 @@ STAGE PLANS: 0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 3490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1861 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 930 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 930 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1745 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 930 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1219,29 +1261,32 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) - outputColumnNames: p_mfgr, p_name, p_size - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: avg(p_size) - keys: p_mfgr (type: string), p_name (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) + outputColumnNames: p_mfgr, p_name, p_size + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(p_size) + keys: p_mfgr (type: string), p_name (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct) Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1692 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/subquery_multiinsert.q.out ql/src/test/results/clientpositive/subquery_multiinsert.q.out index 3c2fab6..6f44cb0 100644 --- ql/src/test/results/clientpositive/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/subquery_multiinsert.q.out @@ -26,7 +26,7 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_5 RUN: Stage-0:DDL -Warning: Shuffle Join JOIN[29][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain from src b INSERT OVERWRITE TABLE src_4 @@ -56,100 +56,78 @@ INSERT OVERWRITE TABLE src_5 order by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-2, Stage-10 - Stage-5 depends on stages: Stage-4 - Stage-6 depends on stages: Stage-5 - Stage-1 depends on stages: Stage-6 - Stage-7 depends on stages: Stage-1 Stage-10 is a root stage + Stage-2 depends on stages: Stage-10 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 + Stage-6 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-6 + Stage-7 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + alias: s1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '2') and key is null) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + aggregations: count() mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 = 0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-3 - Stats-Aggr Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-4 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + alias: b + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TableScan Reduce Output Operator sort order: @@ -170,7 +148,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-5 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -218,7 +196,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-6 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -251,55 +229,77 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_5 - Stage: Stage-7 + Stage: Stage-5 Stats-Aggr Operator - Stage: Stage-10 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '2') and key is null) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 = 0) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + + Stage: Stage-7 + Stats-Aggr Operator -Warning: Shuffle Join JOIN[29][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b INSERT OVERWRITE TABLE src_4 select * @@ -336,15 +336,15 @@ POSTHOOK: Lineage: src_4.key EXPRESSION [(src)b.FieldSchema(name:key, type:strin POSTHOOK: Lineage: src_4.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] -RUN: Stage-2:MAPRED RUN: Stage-10:MAPRED -RUN: Stage-0:MOVE -RUN: Stage-4:MAPRED -RUN: Stage-3:STATS -RUN: Stage-5:MAPRED +RUN: Stage-2:MAPRED +RUN: Stage-3:MAPRED RUN: Stage-6:MAPRED +RUN: Stage-4:MAPRED +RUN: Stage-0:MOVE RUN: Stage-1:MOVE RUN: Stage-7:STATS +RUN: Stage-5:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 @@ -491,8 +491,8 @@ POSTHOOK: Input: default@src_5 199 val_199 199 val_199 2 val_2 -Warning: Map Join MAPJOIN[103][bigTable=?] in task 'Stage-12:MAPRED' is a cross product -Warning: Shuffle Join JOIN[29][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product +Warning: Map Join MAPJOIN[109][bigTable=b] in task 'Stage-13:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain from src b INSERT OVERWRITE TABLE src_4 @@ -523,18 +523,18 @@ INSERT OVERWRITE TABLE src_5 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-10 is a root stage - Stage-13 depends on stages: Stage-10, Stage-14 , consists of Stage-16, Stage-4 - Stage-16 has a backup stage: Stage-4 - Stage-12 depends on stages: Stage-16 - Stage-15 depends on stages: Stage-4, Stage-12 - Stage-6 depends on stages: Stage-15 - Stage-1 depends on stages: Stage-6 - Stage-7 depends on stages: Stage-1 - Stage-4 - Stage-17 is a root stage - Stage-14 depends on stages: Stage-17 - Stage-0 depends on stages: Stage-14 - Stage-3 depends on stages: Stage-0 + Stage-14 depends on stages: Stage-10 , consists of Stage-17, Stage-2 + Stage-17 has a backup stage: Stage-2 + Stage-13 depends on stages: Stage-17 + Stage-15 depends on stages: Stage-2, Stage-13 + Stage-12 depends on stages: Stage-15 + Stage-0 depends on stages: Stage-12 + Stage-7 depends on stages: Stage-0 + Stage-16 depends on stages: Stage-2, Stage-13 + Stage-4 depends on stages: Stage-16 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 + Stage-2 STAGE PLANS: Stage: Stage-10 @@ -582,17 +582,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-13 + Stage: Stage-14 Conditional Operator - Stage: Stage-16 + Stage: Stage-17 Map Reduce Local Work Alias -> Map Local Tables: - $INTNAME1 + $INTNAME Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $INTNAME1 + $INTNAME TableScan HashTable Sink Operator condition expressions: @@ -602,10 +602,11 @@ STAGE PLANS: 0 1 - Stage: Stage-12 + Stage: Stage-13 Map Reduce Map Operator Tree: TableScan + alias: b Map Join Operator condition map: Left Semi Join 0 to 1 @@ -622,12 +623,95 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work Stage: Stage-15 Map Reduce Local Work Alias -> Map Local Tables: + sq_1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + sq_1:a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 _col0 (type: string), _col1 (type: string) + + Stage: Stage-12 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string), value (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + + Stage: Stage-7 + Stats-Aggr Operator + + Stage: Stage-16 + Map Reduce Local Work + Alias -> Map Local Tables: sq_2:s1 Fetch Operator limit: -1 @@ -651,7 +735,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-6 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -704,17 +788,25 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_5 - Stage: Stage-7 + Stage: Stage-5 Stats-Aggr Operator - Stage: Stage-4 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + alias: b + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TableScan Reduce Output Operator sort order: @@ -735,93 +827,8 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-17 - Map Reduce Local Work - Alias -> Map Local Tables: - sq_1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - sq_1:a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 _col0 (type: string), _col1 (type: string) - - Stage: Stage-14 - Map Reduce - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Left Semi Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string), value (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-3 - Stats-Aggr Operator - -Warning: Map Join MAPJOIN[103][bigTable=?] in task 'Stage-12:MAPRED' is a cross product -Warning: Shuffle Join JOIN[29][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product +Warning: Map Join MAPJOIN[109][bigTable=b] in task 'Stage-13:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b INSERT OVERWRITE TABLE src_4 select * @@ -859,17 +866,17 @@ POSTHOOK: Lineage: src_4.value EXPRESSION [(src)b.FieldSchema(name:value, type:s POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-10:MAPRED +RUN: Stage-14:CONDITIONAL RUN: Stage-17:MAPREDLOCAL -RUN: Stage-14:MAPRED -RUN: Stage-0:MOVE -RUN: Stage-13:CONDITIONAL -RUN: Stage-3:STATS +RUN: Stage-13:MAPRED +RUN: Stage-15:MAPREDLOCAL RUN: Stage-16:MAPREDLOCAL RUN: Stage-12:MAPRED -RUN: Stage-15:MAPREDLOCAL -RUN: Stage-6:MAPRED +RUN: Stage-4:MAPRED +RUN: Stage-0:MOVE RUN: Stage-1:MOVE RUN: Stage-7:STATS +RUN: Stage-5:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 diff --git ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 43a3256..c835562 100644 --- ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -83,30 +83,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE TableScan alias: src11 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: key1 (type: string), value1 (type: string) - sort order: ++ - Map-reduce partition columns: key1 (type: string), value1 (type: string) + Filter Operator + predicate: (key1 is not null and value1 is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key1 (type: string), value1 (type: string) + sort order: ++ + Map-reduce partition columns: key1 (type: string), value1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -115,17 +118,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 661 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -152,31 +155,34 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -185,17 +191,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -250,7 +256,7 @@ STAGE PLANS: PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (_wcol0 <= 2) (type: boolean) + predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col2 (type: string), _col5 (type: int) @@ -286,21 +292,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash + Select Operator + expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-3 Map Reduce @@ -314,12 +323,15 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p_size (type: int), p_mfgr (type: string) - sort order: ++ - Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) + Filter Operator + predicate: (p_size is not null and p_mfgr is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p_name (type: string) + Reduce Output Operator + key expressions: p_size (type: int), p_mfgr (type: string) + sort order: ++ + Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p_name (type: string) Reduce Operator Tree: Join Operator condition map: @@ -391,7 +403,7 @@ STAGE PLANS: PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (_wcol0 <= 2) (type: boolean) + predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col2 (type: string), _col5 (type: int) @@ -427,21 +439,24 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash + Select Operator + expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-1 Map Reduce @@ -449,12 +464,15 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: p_size (type: int), p_mfgr (type: string) - sort order: ++ - Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) + Filter Operator + predicate: (p_size is not null and p_mfgr is not null) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: p_name (type: string) + Reduce Output Operator + key expressions: p_size (type: int), p_mfgr (type: string) + sort order: ++ + Map-reduce partition columns: p_size (type: int), p_mfgr (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: p_name (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -524,37 +542,37 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -568,17 +586,20 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 8 Data size: 1603 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -587,17 +608,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 881 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -636,35 +657,41 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -674,33 +701,33 @@ STAGE PLANS: key expressions: _col2 (type: bigint), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col2 (type: bigint), _col1 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: bigint) + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: bigint), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 condition expressions: - 0 {VALUE._col0} {KEY.reducesinkkey1} {VALUE._col1} + 0 {VALUE._col0} {KEY.reducesinkkey1} {KEY.reducesinkkey0} 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1542 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 3093 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -713,23 +740,23 @@ STAGE PLANS: alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: value, key - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: value (type: string), key (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -737,22 +764,25 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint), _col1 (type: string) - mode: hash + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col2 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: bigint), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/subquery_views.q.out ql/src/test/results/clientpositive/subquery_views.q.out index 6c82a59..c4a7656 100644 --- ql/src/test/results/clientpositive/subquery_views.q.out +++ ql/src/test/results/clientpositive/subquery_views.q.out @@ -118,19 +118,19 @@ POSTHOOK: Input: default@cv3 POSTHOOK: Input: default@src #### A masked pattern was here #### 80 val_80 1 +96 val_96 1 +92 val_92 1 +9 val_9 1 +87 val_87 1 +86 val_86 1 +85 val_85 1 82 val_82 1 -83 val_83 2 84 val_84 2 -85 val_85 1 -86 val_86 1 -87 val_87 1 -9 val_9 1 -90 val_90 3 -92 val_92 1 95 val_95 2 -96 val_96 1 -97 val_97 2 +83 val_83 2 98 val_98 2 +97 val_97 2 +90 val_90 3 PREHOOK: query: -- join of subquery views select * from cv3 diff --git ql/src/test/results/clientpositive/table_access_keys_stats.q.out ql/src/test/results/clientpositive/table_access_keys_stats.q.out index 165554f..996edd3 100644 --- ql/src/test/results/clientpositive/table_access_keys_stats.q.out +++ ql/src/test/results/clientpositive/table_access_keys_stats.q.out @@ -273,7 +273,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_4 +Operator:JOIN_6 Table:default@t2 Keys:key Table:default@t1 @@ -292,7 +292,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_4 +Operator:JOIN_6 Table:default@t2 Keys:key,val Table:default@t1 @@ -306,7 +306,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_4 +Operator:JOIN_6 Table:default@t2 Keys:key Table:default@t1 @@ -326,7 +326,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:key Table:default@t1 @@ -347,7 +347,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_8 +Operator:JOIN_10 Table:default@t2 Keys:val Table:default@t1 @@ -365,7 +365,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:val Table:default@t1 @@ -386,7 +386,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_8 +Operator:JOIN_10 Table:default@t2 Keys:val Table:default@t1 @@ -407,7 +407,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:key Table:default@t1 @@ -431,7 +431,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_7 +Operator:JOIN_9 Table:default@t2 Keys:val,key Table:default@t1 @@ -468,7 +468,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 #### A masked pattern was here #### -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:key Table:default@t1 @@ -499,11 +499,11 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 #### A masked pattern was here #### -Operator:GBY_14 +Operator:GBY_16 Table:default@t3 Keys:val -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:key Table:default@t1 @@ -537,7 +537,7 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 #### A masked pattern was here #### -Operator:JOIN_6 +Operator:JOIN_8 Table:default@t2 Keys:key Table:default@t1 diff --git ql/src/test/results/clientpositive/tez/auto_join1.q.out ql/src/test/results/clientpositive/tez/auto_join1.q.out index 5c75f40..f1b0d6a 100644 --- ql/src/test/results/clientpositive/tez/auto_join1.q.out +++ ql/src/test/results/clientpositive/tez/auto_join1.q.out @@ -23,7 +23,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Map 1 <- Map 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -31,40 +31,45 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 Map 2 Map Operator Tree: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index e420fd0..f356cd4 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -128,39 +128,45 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Stage: Stage-0 Fetch Operator @@ -199,49 +205,55 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: srcbucket_mapjoin Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(substr(value, 5)) - keys: key (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(substr(value, 5)) + keys: key (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: Group By Operator @@ -249,16 +261,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double) Stage: Stage-0 @@ -294,55 +306,64 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Map 2 Map Operator Tree: TableScan alias: tab_part Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col5, _col6 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col5 (type: int), _col6 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col5, _col6 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(substr(_col6, 5)) - keys: _col5 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col5 (type: int), _col6 (type: string) + outputColumnNames: _col5, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(substr(_col6, 5)) + keys: _col5 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) Map 4 Map Operator Tree: TableScan alias: tab Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reducer 3 Reduce Operator Tree: Group By Operator @@ -350,11 +371,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -365,14 +386,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 150 Data size: 1600 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 150 Data size: 1600 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 150 Data size: 1600 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -412,70 +433,79 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col1 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: x Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(substr(_col1, 5)) - keys: _col0 (type: int) - mode: hash + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(substr(_col1, 5)) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) Map 4 Map Operator Tree: TableScan alias: y Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -483,16 +513,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 700 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double) Stage: Stage-0 @@ -527,52 +557,61 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Map 2 Map Operator Tree: TableScan alias: c Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - condition expressions: - 0 {key} {value} - 1 {value} - 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {key} {value} + 1 {value} + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -604,64 +643,73 @@ STAGE PLANS: TableScan alias: c Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Map 2 Map Operator Tree: TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 3 Map Operator Tree: TableScan alias: y Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -698,49 +746,55 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: srcbucket_mapjoin Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(substr(value, 5)) - keys: key (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(substr(value, 5)) + keys: key (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: Group By Operator @@ -748,16 +802,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Stage: Stage-0 @@ -793,43 +847,49 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {value} - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col0 (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: srcbucket_mapjoin Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE - value expressions: substr(value, 5) (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: substr(value, 5) (type: string) Reducer 3 Reduce Operator Tree: Group By Operator @@ -837,16 +897,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Stage: Stage-0 @@ -881,39 +941,45 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col6 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col1, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) Stage: Stage-0 Fetch Operator @@ -966,39 +1032,45 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Stage: Stage-0 Fetch Operator @@ -1018,7 +1090,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (CUSTOM_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1026,60 +1098,70 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) Map 2 Map Operator Tree: TableScan alias: c Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} - 1 {key} + 0 {key} {value} + 1 keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col10 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col10 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {key} + keys: + 0 _col1 (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col10 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col10 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out index d3f97d8..d8573b5 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out @@ -116,7 +116,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (CUSTOM_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -124,60 +124,70 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) Map 2 Map Operator Tree: TableScan alias: c Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} - 1 {key} + 0 {key} {value} + 1 keys: - 0 _col1 (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col10 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col10 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {key} + keys: + 0 _col1 (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col10 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col10 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Stage: Stage-0 Fetch Operator @@ -230,39 +240,45 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {value} - keys: - 0 UDFToDouble(key) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col5 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int), value (type: string) + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), value (type: string) Stage: Stage-0 Fetch Operator @@ -293,12 +309,12 @@ STAGE PLANS: alias: tab_part Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 1) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 1) and key is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -309,14 +325,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -327,17 +343,17 @@ STAGE PLANS: alias: tab_part Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 2) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 2) and key is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -516,63 +532,69 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {key} - keys: - 0 _col0 (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {key} + keys: + 0 _col0 (type: int) + 1 key (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: tab Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -601,63 +623,69 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {value} - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 266 Data size: 2822 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: tab Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(value) is not null (type: boolean) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 636 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Stage: Stage-0 diff --git ql/src/test/results/clientpositive/tez/cross_join.q.out ql/src/test/results/clientpositive/tez/cross_join.q.out index c639979..1b247b0 100644 --- ql/src/test/results/clientpositive/tez/cross_join.q.out +++ ql/src/test/results/clientpositive/tez/cross_join.q.out @@ -146,21 +146,27 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -170,14 +176,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out index d1fbf2d..5de4348 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out @@ -80,7 +80,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [d1, d2, a]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[10][tables = [d1, d2, a]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -102,23 +102,29 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: value (type: string) Map 4 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: value (type: string) Map 5 Map Operator Tree: TableScan @@ -137,10 +143,10 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 125 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 0 Data size: 125 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) Reducer 3 Reduce Operator Tree: @@ -170,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [a, od1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[16][tables = [a, od1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -199,21 +205,27 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan @@ -408,7 +420,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [ss, od1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[21][tables = [ss, od1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -436,21 +448,27 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan diff --git ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out index 66f7329..0105a7e 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out @@ -77,7 +77,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[12][bigTable=a] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=a] in task 'Map 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -99,32 +99,38 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 0 Data size: 125 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 0 Data size: 125 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) Map 2 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: value (type: string) Map 3 Map Operator Tree: TableScan @@ -159,7 +165,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=a] in task 'Map 4' is a cross product +Warning: Map Join MAPJOIN[23][bigTable=a] in task 'Map 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -188,44 +194,50 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -384,7 +396,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 5' is a cross product +Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Reducer 5' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -411,44 +423,50 @@ STAGE PLANS: TableScan alias: d1 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (1 = 1) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: d2 Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan diff --git ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out index 5da53dc..5e32403 100644 --- ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out @@ -215,13 +215,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (value <> '') (type: boolean) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE tag: 1 Path -> Alias: #### A masked pattern was here #### @@ -368,17 +368,17 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col11 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col11 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/tez/join1.q.out ql/src/test/results/clientpositive/tez/join1.q.out index 9a51f46..b0192f8 100644 --- ql/src/test/results/clientpositive/tez/join1.q.out +++ ql/src/test/results/clientpositive/tez/join1.q.out @@ -31,22 +31,28 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Map 3 Map Operator Tree: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -56,14 +62,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/tez/limit_pushdown.q.out ql/src/test/results/clientpositive/tez/limit_pushdown.q.out index d6406d2..0ffe684 100644 --- ql/src/test/results/clientpositive/tez/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/tez/limit_pushdown.q.out @@ -872,12 +872,15 @@ STAGE PLANS: Limit Number of rows: 2 Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reducer 3 Reduce Operator Tree: Join Operator @@ -887,17 +890,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 4 - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 220 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -931,12 +934,15 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out index cb9728b..9abaa46 100644 --- ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out +++ ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out @@ -91,38 +91,44 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: dec (type: decimal(4,0)) - sort order: + - Map-reduce partition columns: dec (type: decimal(4,0)) - Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: dec is not null (type: boolean) + Statistics: Num rows: 525 Data size: 58800 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: dec (type: decimal(4,0)) + sort order: + + Map-reduce partition columns: dec (type: decimal(4,0)) + Statistics: Num rows: 525 Data size: 58800 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: t1 Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {dec} - 1 {dec} - keys: - 0 dec (type: decimal(4,2)) - 1 dec (type: decimal(4,0)) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: decimal(4,2)), _col3 (type: decimal(4,0)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: dec is not null (type: boolean) + Statistics: Num rows: 525 Data size: 58800 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {dec} + 1 {dec} + keys: + 0 dec (type: decimal(4,2)) + 1 dec (type: decimal(4,0)) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 577 Data size: 64680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: decimal(4,2)), _col3 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 577 Data size: 64680 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 577 Data size: 64680 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out index 9e2ae8f..9e90ec2 100644 --- ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out @@ -22,59 +22,68 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} + 0 {key} 1 keys: - 0 _col0 (type: string) - 1 key (type: string) + 0 value (type: string) + 1 value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -105,8 +114,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 38 Data size: 7615 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and key is not null) and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -117,7 +126,7 @@ STAGE PLANS: 0 value (type: string) 1 value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 41 Data size: 8376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -128,14 +137,14 @@ STAGE PLANS: 0 _col0 (type: string) 1 key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 45 Data size: 9213 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -145,24 +154,27 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -193,64 +205,73 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {ds} - 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 127 Data size: 25572 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col2} + 0 {key} {ds} 1 keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col2 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: string) + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col2} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) outputColumnNames: _col2 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col2 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 139 Data size: 28129 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string) + outputColumnNames: _col2 + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 34 Data size: 7032 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Map 3 Map Operator Tree: TableScan alias: src1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -258,14 +279,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 69 Data size: 13963 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 3516 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/tez/mrr.q.out ql/src/test/results/clientpositive/tez/mrr.q.out index 9b05cd1..8fcf23e 100644 --- ql/src/test/results/clientpositive/tez/mrr.q.out +++ ql/src/test/results/clientpositive/tez/mrr.q.out @@ -415,22 +415,28 @@ STAGE PLANS: TableScan alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Map 5 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -440,22 +446,22 @@ STAGE PLANS: 0 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col4, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: string), _col5 (type: string) outputColumnNames: _col4, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(DISTINCT _col5) keys: _col4 (type: string), _col5 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -463,25 +469,25 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: bigint) sort order: + - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -824,9 +830,9 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -834,69 +840,74 @@ STAGE PLANS: TableScan alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Map 2 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col4, _col5 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col4 (type: string), _col5 (type: string) + outputColumnNames: _col4, _col5 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(DISTINCT _col5) + keys: _col4 (type: string), _col5 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE + Map 4 Map Operator Tree: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {key} {value} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col4, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col4 (type: string), _col5 (type: string) - outputColumnNames: _col4, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(DISTINCT _col5) - keys: _col4 (type: string), _col5 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col1:0._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: bigint) sort order: + - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 4 + Reducer 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 3145 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 1546 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1282,64 +1293,73 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Map 6 Map Operator Tree: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Map 9 Map Operator Tree: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reducer 10 Reduce Operator Tree: Group By Operator @@ -1347,30 +1367,30 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 > 1) (type: boolean) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: bigint) sort order: + - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 11 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1379,27 +1399,27 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: bigint) sort order: + - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1412,25 +1432,25 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 30 Data size: 6171 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3084 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 30 Data size: 6171 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3084 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 30 Data size: 6171 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3084 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Reducer 5 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: bigint), VALUE._col3 (type: string), VALUE._col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 30 Data size: 6171 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3084 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 30 Data size: 6171 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3084 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1442,27 +1462,27 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: bigint) sort order: + - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 8 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Stage: Stage-0 @@ -1672,49 +1692,55 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {key} {value} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 2 Map Operator Tree: TableScan alias: src Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reducer 3 Reduce Operator Tree: Group By Operator @@ -1722,16 +1748,16 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Stage: Stage-0 diff --git ql/src/test/results/clientpositive/tez/subquery_in.q.out ql/src/test/results/clientpositive/tez/subquery_in.q.out index 6e6becf..e96485b 100644 --- ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -839,33 +839,36 @@ STAGE PLANS: alias: li Statistics: Num rows: 756 Data size: 12099 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (l_linenumber = 1) (type: boolean) - Statistics: Num rows: 378 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + predicate: (l_partkey is not null and (l_linenumber = 1)) (type: boolean) + Statistics: Num rows: 189 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: l_partkey (type: int) sort order: + Map-reduce partition columns: l_partkey (type: int) - Statistics: Num rows: 378 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 189 Data size: 3024 Basic stats: COMPLETE Column stats: NONE value expressions: l_orderkey (type: int), l_suppkey (type: int) Map 4 Map Operator Tree: TableScan alias: lineitem Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: l_partkey (type: int) - outputColumnNames: l_partkey - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: l_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3024 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: l_partkey (type: int) + outputColumnNames: l_partkey + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: l_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan @@ -897,12 +900,12 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} {VALUE._col1} outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 1663 Data size: 6653 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1663 Data size: 6653 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: int) Reducer 3 Reduce Operator Tree: @@ -913,14 +916,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col2} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1829 Data size: 7318 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -931,16 +934,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1512 Data size: 6049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 756 Data size: 3024 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/tez_union.q.out ql/src/test/results/clientpositive/tez/tez_union.q.out index a971039..d1511f2 100644 --- ql/src/test/results/clientpositive/tez/tez_union.q.out +++ ql/src/test/results/clientpositive/tez/tez_union.q.out @@ -24,38 +24,43 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s1 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col1 Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 3 Map Operator Tree: TableScan alias: s3 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -160,46 +165,54 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Map 5 Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Map 7 Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Map 8 Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Reducer 3 Reduce Operator Tree: Join Operator @@ -208,9 +221,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -304,68 +317,75 @@ STAGE PLANS: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {_col0} - keys: - 0 key (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col4 - Select Operator - expressions: _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {_col0} + keys: + 0 key (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col4 + Select Operator + expressions: _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 4 Map Operator Tree: TableScan alias: src - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {_col0} - keys: - 0 key (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col4 - Select Operator - expressions: _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {_col0} + keys: + 0 key (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col4 + Select Operator + expressions: _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 3 Vertex: Union 3 @@ -459,191 +479,212 @@ STAGE PLANS: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 10 Map Operator Tree: TableScan alias: s5 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: s2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} - 1 {_col0} - 2 {key} - keys: - 0 key (type: string) - 1 _col0 (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col4, _col5 - Select Operator - expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col4, _col5 + Select Operator + expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) Map 5 Map Operator Tree: TableScan alias: s3 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan alias: s7 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: s6 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} - 1 {_col0} - 2 {key} - keys: - 0 key (type: string) - 1 _col0 (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col4, _col5 - Select Operator - expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col4, _col5 + Select Operator + expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) Map 8 Map Operator Tree: TableScan alias: s8 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 9 Map Operator Tree: TableScan alias: s4 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - condition expressions: - 0 {key} - 1 {_col0} - 2 {key} - keys: - 0 key (type: string) - 1 _col0 (type: string) - 2 key (type: string) - outputColumnNames: _col0, _col4, _col5 - Select Operator - expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col4, _col5 + Select Operator + expressions: _col0 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 415 Data size: 42193 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 204 Data size: 21093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 415 Data size: 42193 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 204 Data size: 21093 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -818,74 +859,84 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 3 Map Operator Tree: TableScan alias: s3 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: s4 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Map 5 Map Operator Tree: TableScan alias: s5 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/udf_case_column_pruning.q.out ql/src/test/results/clientpositive/udf_case_column_pruning.q.out index ecf016f..f6f736a 100644 --- ql/src/test/results/clientpositive/udf_case_column_pruning.q.out +++ ql/src/test/results/clientpositive/udf_case_column_pruning.q.out @@ -30,19 +30,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -51,11 +57,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: CASE (_col0) WHEN ('1') THEN (2) WHEN ('3') THEN (4) ELSE (5) END (type: int) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -70,18 +76,18 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int) sort order: + - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/union20.q.out ql/src/test/results/clientpositive/union20.q.out index 715f47d..07e810f 100644 --- ql/src/test/results/clientpositive/union20.q.out +++ ql/src/test/results/clientpositive/union20.q.out @@ -37,17 +37,20 @@ STAGE PLANS: TableScan alias: s3 Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Filter Operator + predicate: 'tst1' is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -70,57 +73,57 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan alias: s4 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and key is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan Union - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 2075 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1274 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -130,14 +133,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Statistics: Num rows: 18 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Statistics: Num rows: 18 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Statistics: Num rows: 18 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -149,17 +152,20 @@ STAGE PLANS: TableScan alias: s1 Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Filter Operator + predicate: 'tst1' is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/union24.q.out ql/src/test/results/clientpositive/union24.q.out index f1df435..dda55ca 100644 --- ql/src/test/results/clientpositive/union24.q.out +++ ql/src/test/results/clientpositive/union24.q.out @@ -794,13 +794,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key < 10)) (type: boolean) + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: count (type: bigint) TableScan @@ -809,13 +809,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key < 10)) (type: boolean) + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE tag: 0 Path -> Alias: #### A masked pattern was here #### @@ -920,11 +920,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -949,17 +949,17 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -988,17 +988,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1027,17 +1027,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1368,13 +1368,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key < 10)) (type: boolean) + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE tag: 1 TableScan alias: a @@ -1382,13 +1382,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + predicate: (key is not null and (key < 10)) (type: boolean) + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE tag: 0 Path -> Alias: #### A masked pattern was here #### @@ -1493,17 +1493,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1531,7 +1531,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) Path -> Alias: @@ -1566,11 +1566,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 134 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 134 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1595,17 +1595,17 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1634,17 +1634,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1673,17 +1673,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/union26.q.out ql/src/test/results/clientpositive/union26.q.out index e191c6d..e184664 100644 --- ql/src/test/results/clientpositive/union26.q.out +++ ql/src/test/results/clientpositive/union26.q.out @@ -58,20 +58,26 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -80,11 +86,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -112,22 +118,22 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Select Operator expressions: array(1,2,3) (type: array) @@ -144,41 +150,41 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) TableScan Union - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 121 Data size: 18017 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 89 Data size: 14820 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -186,14 +192,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 8934 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 44 Data size: 7326 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 8934 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 44 Data size: 7326 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 60 Data size: 8934 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 44 Data size: 7326 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/union32.q.out ql/src/test/results/clientpositive/union32.q.out index ffd45ba..35363c1 100644 --- ql/src/test/results/clientpositive/union32.q.out +++ ql/src/test/results/clientpositive/union32.q.out @@ -161,19 +161,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -182,11 +188,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -205,28 +211,28 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -312,19 +318,25 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -333,11 +345,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -350,14 +362,14 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -370,14 +382,14 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -463,19 +475,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -484,11 +502,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col4 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(UDFToDouble(_col4)) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -507,28 +525,28 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -614,19 +632,25 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE TableScan alias: b Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -635,11 +659,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col4 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToDouble(_col4) (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -652,14 +676,14 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -672,14 +696,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/union34.q.out ql/src/test/results/clientpositive/union34.q.out index b21287c..d5224af 100644 --- ql/src/test/results/clientpositive/union34.q.out +++ ql/src/test/results/clientpositive/union34.q.out @@ -91,17 +91,20 @@ STAGE PLANS: TableScan alias: src10_1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col1} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-2 Map Reduce @@ -120,15 +123,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src10_4 @@ -144,59 +147,62 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src10_2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -294,28 +300,34 @@ STAGE PLANS: TableScan alias: src10_2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE TableScan alias: src10_1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -324,11 +336,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -341,15 +353,15 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src10_3 @@ -365,15 +377,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src10_4 @@ -389,24 +401,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 322 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/union_remove_12.q.out ql/src/test/results/clientpositive/union_remove_12.q.out index c897d61..c9c9b22 100644 --- ql/src/test/results/clientpositive/union_remove_12.q.out +++ ql/src/test/results/clientpositive/union_remove_12.q.out @@ -139,13 +139,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-9 Map Reduce @@ -153,29 +156,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/union_remove_13.q.out ql/src/test/results/clientpositive/union_remove_13.q.out index eaf74a6..01b3bac 100644 --- ql/src/test/results/clientpositive/union_remove_13.q.out +++ ql/src/test/results/clientpositive/union_remove_13.q.out @@ -162,13 +162,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-9 Map Reduce @@ -176,29 +179,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/union_remove_14.q.out ql/src/test/results/clientpositive/union_remove_14.q.out index 7f15947..1681872 100644 --- ql/src/test/results/clientpositive/union_remove_14.q.out +++ ql/src/test/results/clientpositive/union_remove_14.q.out @@ -141,13 +141,16 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-9 Map Reduce @@ -155,29 +158,32 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {val} - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {val} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/union_remove_23.q.out ql/src/test/results/clientpositive/union_remove_23.q.out index 8e714df..b50773f 100644 --- ql/src/test/results/clientpositive/union_remove_23.q.out +++ ql/src/test/results/clientpositive/union_remove_23.q.out @@ -127,19 +127,25 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE TableScan alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -148,17 +154,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false table: @@ -174,7 +180,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/union_top_level.q.out ql/src/test/results/clientpositive/union_top_level.q.out index ccb0b93..a01571c 100644 --- ql/src/test/results/clientpositive/union_top_level.q.out +++ ql/src/test/results/clientpositive/union_top_level.q.out @@ -233,20 +233,26 @@ STAGE PLANS: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -255,14 +261,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -276,16 +282,16 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -298,28 +304,28 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 2020 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2060 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -331,20 +337,26 @@ STAGE PLANS: TableScan alias: s1 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE TableScan alias: s2 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: @@ -353,14 +365,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 3196 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -374,16 +386,16 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1010 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1030 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out index 19e0b36..71a3def 100644 --- ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out @@ -41,8 +41,8 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) - Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + predicate: (cint is not null and (cint = 6981)) (type: boolean) + Statistics: Num rows: 3072 Data size: 541265 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {cdecimal1} @@ -58,8 +58,8 @@ STAGE PLANS: alias: r Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) - Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE + predicate: (cint is not null and (cint = 6981)) (type: boolean) + Statistics: Num rows: 3072 Data size: 541265 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -70,14 +70,14 @@ STAGE PLANS: 0 cint (type: int) 1 cint (type: int) outputColumnNames: _col1, _col3, _col8, _col9 - Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col9 (type: int), _col1 (type: decimal(20,10)), _col8 (type: decimal(23,14)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out index 055c2ac..33b5b3b 100644 --- ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out @@ -107,25 +107,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -165,25 +168,28 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -233,25 +239,28 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} {value} - 1 {key} {value} - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col4, _col5 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col4, _col5 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 diff --git ql/src/test/results/clientpositive/vectorized_context.q.out ql/src/test/results/clientpositive/vectorized_context.q.out index a65d69d..af238b0 100644 --- ql/src/test/results/clientpositive/vectorized_context.q.out +++ ql/src/test/results/clientpositive/vectorized_context.q.out @@ -113,24 +113,30 @@ STAGE PLANS: TableScan alias: household_demographics Statistics: Num rows: 6075 Data size: 24300 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col2} {_col6} - 1 - keys: - 0 _col1 (type: int) - 1 hd_demo_sk (type: int) + Filter Operator + predicate: hd_demo_sk is not null (type: boolean) + Statistics: Num rows: 3038 Data size: 12152 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col2} {_col6} + 1 + keys: + 0 _col1 (type: int) + 1 hd_demo_sk (type: int) store_sales TableScan alias: store_sales Statistics: Num rows: 6075 Data size: 72736 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {ss_hdemo_sk} {ss_net_profit} - 1 {s_city} - keys: - 0 ss_store_sk (type: int) - 1 s_store_sk (type: int) + Filter Operator + predicate: (ss_store_sk is not null and ss_hdemo_sk is not null) (type: boolean) + Statistics: Num rows: 1519 Data size: 18186 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {ss_hdemo_sk} {ss_net_profit} + 1 {s_city} + keys: + 0 ss_store_sk (type: int) + 1 s_store_sk (type: int) Stage: Stage-4 Map Reduce @@ -138,42 +144,45 @@ STAGE PLANS: TableScan alias: store Statistics: Num rows: 6075 Data size: 615632 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {ss_hdemo_sk} {ss_net_profit} - 1 {s_city} - keys: - 0 ss_store_sk (type: int) - 1 s_store_sk (type: int) - outputColumnNames: _col1, _col2, _col6 - Statistics: Num rows: 6682 Data size: 80009 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 3038 Data size: 307866 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col2} {_col6} - 1 + 0 {ss_hdemo_sk} {ss_net_profit} + 1 {s_city} keys: - 0 _col1 (type: int) - 1 hd_demo_sk (type: int) - outputColumnNames: _col2, _col6 - Statistics: Num rows: 7350 Data size: 88009 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col6 (type: string), _col2 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7350 Data size: 88009 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 1100 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 1100 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 ss_store_sk (type: int) + 1 s_store_sk (type: int) + outputColumnNames: _col1, _col2, _col6 + Statistics: Num rows: 3341 Data size: 338652 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col2} {_col6} + 1 + keys: + 0 _col1 (type: int) + 1 hd_demo_sk (type: int) + outputColumnNames: _col2, _col6 + Statistics: Num rows: 3675 Data size: 372517 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col6 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3675 Data size: 372517 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 10100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 10100 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work Execution mode: vectorized diff --git ql/src/test/results/clientpositive/vectorized_mapjoin.q.out ql/src/test/results/clientpositive/vectorized_mapjoin.q.out index df436d0..6a085f6 100644 --- ql/src/test/results/clientpositive/vectorized_mapjoin.q.out +++ ql/src/test/results/clientpositive/vectorized_mapjoin.q.out @@ -23,13 +23,16 @@ STAGE PLANS: TableScan alias: t1 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 - 1 {cint} - keys: - 0 cint (type: int) - 1 cint (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {cint} + keys: + 0 cint (type: int) + 1 cint (type: int) Stage: Stage-2 Map Reduce @@ -37,30 +40,33 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {cint} - 1 {cint} - keys: - 0 cint (type: int) - 1 cint (type: int) - outputColumnNames: _col2, _col16 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col16 (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {cint} + 1 {cint} + keys: + 0 cint (type: int) + 1 cint (type: int) outputColumnNames: _col2, _col16 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2), max(_col16), min(_col2), avg((_col2 + _col16)) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col16 (type: int) + outputColumnNames: _col2, _col16 + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2), max(_col16), min(_col2), avg((_col2 + _col16)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct) Local Work: Map Reduce Local Work Execution mode: vectorized diff --git ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out index 0246dd4..ac2dfb5 100644 --- ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out +++ ql/src/test/results/clientpositive/vectorized_nested_mapjoin.q.out @@ -22,24 +22,30 @@ STAGE PLANS: TableScan alias: v1 Statistics: Num rows: 23577 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {csmallint} {cdouble} - 1 {ctinyint} - keys: - 0 ctinyint (type: tinyint) - 1 ctinyint (type: tinyint) + Filter Operator + predicate: (ctinyint is not null and csmallint is not null) (type: boolean) + Statistics: Num rows: 5895 Data size: 94321 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {csmallint} {cdouble} + 1 {ctinyint} + keys: + 0 ctinyint (type: tinyint) + 1 ctinyint (type: tinyint) v3 TableScan alias: v3 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - condition expressions: - 0 {_col1} - 1 - keys: - 0 _col0 (type: smallint) - 1 csmallint (type: smallint) + Filter Operator + predicate: csmallint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 + keys: + 0 _col0 (type: smallint) + 1 csmallint (type: smallint) Stage: Stage-2 Map Reduce @@ -47,48 +53,51 @@ STAGE PLANS: TableScan alias: v2 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {ctinyint} {csmallint} {cdouble} - 1 {ctinyint} - keys: - 0 ctinyint (type: tinyint) - 1 ctinyint (type: tinyint) - outputColumnNames: _col0, _col1, _col5, _col14 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col0 = _col14) (type: boolean) - Statistics: Num rows: 51869 Data size: 207477 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: smallint), _col5 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51869 Data size: 207477 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col1} - 1 - keys: - 0 _col0 (type: smallint) - 1 csmallint (type: smallint) - outputColumnNames: _col1 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double) + Filter Operator + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {ctinyint} {csmallint} {cdouble} + 1 {ctinyint} + keys: + 0 ctinyint (type: tinyint) + 1 ctinyint (type: tinyint) + outputColumnNames: _col0, _col1, _col5, _col14 + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = _col14) (type: boolean) + Statistics: Num rows: 25935 Data size: 103741 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25935 Data size: 103741 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col1} + 1 + keys: + 0 _col0 (type: smallint) + 1 csmallint (type: smallint) outputColumnNames: _col1 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(_col1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double) + outputColumnNames: _col1 + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col1) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) Local Work: Map Reduce Local Work Execution mode: vectorized diff --git ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out index 17683cd..e2c3295 100644 --- ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out +++ ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out @@ -18,19 +18,25 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cint (type: int) - sort order: + - Map-reduce partition columns: cint (type: int) - Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: cint (type: int) + sort order: + + Map-reduce partition columns: cint (type: int) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE TableScan alias: t1 Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: cint (type: int) - sort order: + - Map-reduce partition columns: cint (type: int) - Statistics: Num rows: 94309 Data size: 377237 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: cint (type: int) + sort order: + + Map-reduce partition columns: cint (type: int) + Statistics: Num rows: 47155 Data size: 188620 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reduce Operator Tree: Join Operator @@ -40,11 +46,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col2, _col16 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col16 (type: int) outputColumnNames: _col2, _col16 - Statistics: Num rows: 103739 Data size: 414960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 51870 Data size: 207482 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2), max(_col16), min(_col2), avg((_col2 + _col16)) mode: hash diff --git ql/src/test/results/compiler/plan/join1.q.xml ql/src/test/results/compiler/plan/join1.q.xml index 376ba57..e88d5dd 100644 --- ql/src/test/results/compiler/plan/join1.q.xml +++ ql/src/test/results/compiler/plan/join1.q.xml @@ -375,146 +375,217 @@ - - - - - VALUE._col0 - - - value - - - src2 + + + + + + + + VALUE._col0 + + + value + + + src2 + + + + + string + + + + + + + KEY.reducesinkkey0 + + + key + + + src2 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + _col0 + + + + + + + + 1 + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + _col0 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + string + + + escape.delim + \ + + + + + + + + + RS_5 + + + + - - - - string + + + + + + + + VALUE._col0 + + + src2 + + + + + + string + + - - - - - - KEY.reducesinkkey0 - - - key - - - src2 - - - - - - - + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string + + + + + + + + + key + + + src2 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 - - - - - - - _col0 - - - - - - - - 1 - - - - - + + - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - _col0 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - string - - - escape.delim - \ + + + + boolean @@ -523,7 +594,7 @@ - RS_3 + FIL_9 @@ -534,9 +605,25 @@ - + - VALUE._col0 + key + + + src2 + + + + + + string + + + + + + + value src2 @@ -602,36 +689,10 @@ - - - key - - - src2 - - - - - - string - - + - - - value - - - src2 - - - - - - string - - + @@ -645,7 +706,7 @@ src2 - + bigint @@ -685,126 +746,174 @@ - - - - - KEY.reducesinkkey0 - - - key - - - src1 - - - + + + + + + + + KEY.reducesinkkey0 + + + key + + + src1 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + + + + escape.delim + \ + + + + + + + + + RS_3 + + + + - - - + + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string + + + + + + + + + key + + + src1 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 - - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat + + - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - - - - escape.delim - \ - - + + - RS_2 + FIL_10 @@ -812,7 +921,26 @@ - + + + + + + key + + + src1 + + + + + + string + + + + + @@ -855,20 +983,7 @@ - - - key - - - src1 - - - - - - string - - + @@ -898,7 +1013,7 @@ src1 - + bigint @@ -1105,7 +1220,7 @@ - FS_6 + FS_8 @@ -1210,7 +1325,7 @@ - SEL_5 + SEL_7 @@ -1403,7 +1518,7 @@ - JOIN_4 + JOIN_6 diff --git ql/src/test/results/compiler/plan/join2.q.xml ql/src/test/results/compiler/plan/join2.q.xml index ad16e9c..11c44c7 100644 --- ql/src/test/results/compiler/plan/join2.q.xml +++ ql/src/test/results/compiler/plan/join2.q.xml @@ -501,7 +501,7 @@ - RS_6 + RS_9 @@ -558,7 +558,7 @@ - TS_12 + TS_19 @@ -567,7 +567,7 @@ - + _col0 @@ -583,7 +583,7 @@ - + _col4 @@ -608,163 +608,255 @@ - - - - - VALUE._col1 - - - value - - - src3 - - - + + + + + + + + VALUE._col1 + + + value + + + src3 + + + + + + + + KEY.reducesinkkey0 + + + + + + + key + + + src3 + + + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + UDFToDouble + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + double + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + _col1 + + + + + + + + 1 + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + _col1 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + string + + + escape.delim + \ + + + + + + + + + RS_11 + + + + - - - - KEY.reducesinkkey0 - - - + + + + - - - key + + + VALUE._col1 src3 - + + + string + - - - - - - org.apache.hadoop.hive.ql.udf.UDFToDouble - - - UDFToDouble - - - - - - - - - + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - double + + + + + + + + + + + + + key + + + src3 + + + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + UDFToDouble + + + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 - - - - - - - _col1 - - - - - - - - 1 - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + - - - - columns - _col1 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - string - - - escape.delim - \ + + + + boolean @@ -773,7 +865,7 @@ - RS_7 + FIL_16 @@ -784,9 +876,25 @@ - + - VALUE._col1 + key + + + src3 + + + + + + string + + + + + + + value src3 @@ -852,36 +960,10 @@ - - - key - - - src3 - - - - - - string - - + - - - value - - - src3 - - - - - - string - - + @@ -895,7 +977,7 @@ src3 - + bigint @@ -1158,7 +1240,7 @@ - FS_10 + FS_14 @@ -1263,7 +1345,7 @@ - SEL_9 + SEL_13 @@ -1472,7 +1554,7 @@ - JOIN_8 + JOIN_12 @@ -1705,129 +1787,177 @@ - - - - - KEY.reducesinkkey0 - - - key - - - src2 - - - - - - - - - - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string - - - - - - - -1 - - - 1 - - - -1 - - - + + + + + + + + KEY.reducesinkkey0 + + + key + + + src2 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + + + + + 1 + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + + + + escape.delim + \ + + + + + + + + + RS_6 + + - reducesinkkey0 - - - - - - - - - - - 1 - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + - - - - columns - - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - - - - escape.delim - \ + + + + + + + + + + + + + + + + + key + + + src2 + + + + + + + + + + + - RS_4 + FIL_15 @@ -1835,7 +1965,26 @@ - + + + + + + key + + + src2 + + + + + + string + + + + + @@ -1878,20 +2027,7 @@ - - - key - - - src2 - - - - - - string - - + @@ -1921,7 +2057,7 @@ src2 - + bigint @@ -1957,126 +2093,174 @@ - - - - - KEY.reducesinkkey0 - - - key - - - src1 - - - + + + + + + + + KEY.reducesinkkey0 + + + key + + + src1 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + + + + escape.delim + \ + + + + + + + + + RS_4 + + + + - - - + + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string + + + + + + + + + key + + + src1 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 - - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat + + - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - - - - escape.delim - \ - - + + - RS_3 + FIL_17 @@ -2084,7 +2268,26 @@ - + + + + + + key + + + src1 + + + + + + string + + + + + @@ -2127,20 +2330,7 @@ - - - key - - - src1 - - - - - - string - - + @@ -2170,7 +2360,7 @@ src1 - + bigint @@ -2325,30 +2515,68 @@ - + + + + + + + + + #### A masked pattern was here #### + + + + + + + 1 + + + + + + 1 + + + + + FS_18 + + + + + + + + + + + + - - - - #### A masked pattern was here #### + + + + + + + + + + + + + + + + - - - - - 1 - - - - - - 1 - - FS_11 + FIL_8 @@ -2506,7 +2734,7 @@ - JOIN_5 + JOIN_7 @@ -2537,7 +2765,16 @@ - + + + + + + + + + + diff --git ql/src/test/results/compiler/plan/join3.q.xml ql/src/test/results/compiler/plan/join3.q.xml index 489a0d1..6fde4e0 100644 --- ql/src/test/results/compiler/plan/join3.q.xml +++ ql/src/test/results/compiler/plan/join3.q.xml @@ -440,124 +440,176 @@ - - - - - KEY.reducesinkkey0 - - - key - - - src2 - - - - - string - - + + + + + + + + KEY.reducesinkkey0 + + + key + + + src2 + + + + + string + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + + + + + 1 + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + + + + escape.delim + \ + + + + + + + + + RS_6 + + + + - - - + + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string + + + + + + + + + key + + + src2 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 + + - - - - - - - - - - 1 - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - - - - escape.delim - \ + + + + boolean @@ -566,7 +618,7 @@ - RS_4 + FIL_12 @@ -574,7 +626,26 @@ - + + + + + + key + + + src2 + + + + + + string + + + + + @@ -617,20 +688,7 @@ - - - key - - - src2 - - - - - - string - - + @@ -660,7 +718,7 @@ src2 - + bigint @@ -700,151 +758,218 @@ - - - - - VALUE._col0 - - - value - - - src3 - - - - - - - - KEY.reducesinkkey0 - - - key - - - src3 - - - - - - - - - - - - - - - + + + + + + + + VALUE._col0 + + + value + + + src3 + + + + + + + + KEY.reducesinkkey0 + + + key + + + src3 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + _col0 + + + + + + + + 2 + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + _col0 + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + string + + + escape.delim + \ + + + + + + + + + RS_8 + + - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat + - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + + + + + + + + + VALUE._col0 + + + src3 + + + + + + string + + - - columns.types - string + + + + + + + + + + + + + + + + key + + + src3 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 + + - - - - - - _col0 - - - - - - - - 2 - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - _col0 - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - string - - - escape.delim - \ - - + + - RS_5 + FIL_13 @@ -855,9 +980,25 @@ - + - VALUE._col0 + key + + + src3 + + + + + + string + + + + + + + value src3 @@ -923,36 +1064,10 @@ - - - key - - - src3 - - - - - - string - - + - - - value - - - src3 - - - - - - string - - + @@ -966,7 +1081,7 @@ src3 - + bigint @@ -1002,126 +1117,174 @@ - - - - - KEY.reducesinkkey0 - - - key - - - src1 - - - + + + + + + + + KEY.reducesinkkey0 + + + key + + + src1 + + + + + + + + + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + reducesinkkey0 + + + serialization.lib + org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe + + + serialization.sort.order + + + + + columns.types + string + + + + + + + -1 + + + 1 + + + -1 + + + + + reducesinkkey0 + + + + + + + + + + + + + + + + org.apache.hadoop.mapred.SequenceFileInputFormat + + + org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + + + + columns + + + + serialization.lib + org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + + columns.types + + + + escape.delim + \ + + + + + + + + + RS_4 + + + + - - - + + + + + + - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - - - - - columns - reducesinkkey0 - - - serialization.lib - org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe - - - serialization.sort.order - + - - - columns.types - string + + + + + + + + + key + + + src1 + + + + + - - - - -1 - - - 1 - - - -1 - - - - - reducesinkkey0 - - - - - - - - - - - - - - - - org.apache.hadoop.mapred.SequenceFileInputFormat - - - org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + - - - - columns - - - - serialization.lib - org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - - columns.types - - - - escape.delim - \ - - + + - RS_3 + FIL_14 @@ -1129,7 +1292,26 @@ - + + + + + + key + + + src1 + + + + + + string + + + + + @@ -1172,20 +1354,7 @@ - - - key - - - src1 - - - - - - string - - + @@ -1215,7 +1384,7 @@ src1 - + bigint @@ -1425,7 +1594,7 @@ - FS_8 + FS_11 @@ -1530,7 +1699,7 @@ - SEL_7 + SEL_10 @@ -1757,7 +1926,7 @@ - JOIN_6 + JOIN_9