diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 753f25b3b5..557a422272 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -1355,6 +1355,35 @@ public String toString() { return getName() + '[' + getIdentifier() + ']'; } + /** For debugging purposes only, a quick way to get a picture of the tree. */ + public String getTree(String indent) { + String tree = indent + getName() + '[' + getIdentifier() + ']' + '\n'; + for (Operator child : childOperators) { + tree += child.getTree(indent + '\t'); + } + return tree; + } + + /** For debugging purposes only, a quick way to get the elements of the tree. */ + public List> getTreeElements() { + List> treeElements = new ArrayList<>(); + treeElements.add(this); + for (Operator child : childOperators) { + treeElements.addAll(child.getTreeElements()); + } + return treeElements; + } + + /** For debugging purposes only, a quick way to get the elements of the tree. */ + public List getTreeElementDescs() { + List treeElementDescs = new ArrayList<>(); + treeElementDescs.add(conf); + for (Operator child : childOperators) { + treeElementDescs.addAll(child.getTreeElementDescs()); + } + return treeElementDescs; + } + public static String toString(Collection top) { StringBuilder builder = new StringBuilder(); Set visited = new HashSet(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index a5fe3bb6a9..f2e816c85c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -22,16 +22,16 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Stack; import com.google.common.base.Predicates; import com.google.common.collect.Iterators; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; /** This optimization tries to remove {@link SelectOperator} from tree which don't do any * processing except forwarding columns from its parent to its children. @@ -73,19 +74,8 @@ private static final Logger LOG = LoggerFactory.getLogger(IdentityProjectRemover.class); @Override public ParseContext transform(ParseContext pctx) throws SemanticException { - // 0. We check the conditions to apply this transformation, - // if we do not meet them we bail out - final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); - final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); - final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); - if(cboEnabled && returnPathEnabled && cboSucceeded) { - return pctx; - } - - // 1. We apply the transformation - Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", - "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); + Map opRules = new LinkedHashMap<>(); + opRules.put(new RuleRegExp("R1", "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); SemanticGraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null)); ArrayList topNodes = new ArrayList(); topNodes.addAll(pctx.getTopOps().values()); @@ -95,18 +85,29 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { private static class ProjectRemover implements SemanticNodeProcessor { + // Do not remove projection before a FileSink / Union, as it may be the only projection before it, that removes the + // unnecessary columns like row__id. + private static final Set DO_NOT_REMOVE_IF_SINGLE_CHILD = + Sets.immutableEnumSet(OperatorType.FILESINK, OperatorType.UNION); + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { - + SelectOperator sel = (SelectOperator)nd; List> parents = sel.getParentOperators(); - if (parents.size() != 1 || parents.get(0) instanceof LateralViewForwardOperator) { + if (parents.size() != 1 || parents.get(0).getType() == OperatorType.LATERALVIEWFORWARD || + parents.get(0).getType() == OperatorType.FILESINK) { // Multi parents, cant handle that. - // Right now, we do not remove projection on top of - // LateralViewForward operators. + // Right now, we do not remove projection on top of LateralViewForward. return null; } + + List> childs = sel.getChildOperators(); + if (childs.size() == 1 && DO_NOT_REMOVE_IF_SINGLE_CHILD.contains(childs.get(0).getType())) { + return null; + } + Operator parent = parents.get(0); if (parent instanceof ReduceSinkOperator && Iterators.any(sel.getChildOperators().iterator(), Predicates.instanceOf(ReduceSinkOperator.class))) { @@ -126,7 +127,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } - if(sel.isIdentitySelect()) { + if (sel.isIdentitySelect()) { parent.removeChildAndAdoptItsChildren(sel); LOG.debug("Identity project remover optimization removed : " + sel); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index da277d058f..8ca4c1ea06 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -195,8 +195,7 @@ public void initialize(HiveConf hiveConf) { transformations.add(new ReduceSinkDeDuplication()); } transformations.add(new NonBlockingOpDeDupProc()); - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEIDENTITYPROJECTREMOVER) - && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEIDENTITYPROJECTREMOVER)) { transformations.add(new IdentityProjectRemover()); } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVELIMITOPTENABLE)) { diff --git ql/src/test/queries/clientpositive/cbo_rp_cast1.q ql/src/test/queries/clientpositive/cbo_rp_cast1.q new file mode 100644 index 0000000000..a3fd707cbe --- /dev/null +++ ql/src/test/queries/clientpositive/cbo_rp_cast1.q @@ -0,0 +1,11 @@ +--! qt:dataset:src +set hive.mapred.mode=nonstrict; +CREATE TABLE dest1_n151(c1 INT, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 INT, c6 STRING, c7 INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86; + +FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86; + +select dest1_n151.* FROM dest1_n151; + diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out index e85c5c9ec6..595cf07cde 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out @@ -106,27 +106,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) TableScan alias: subq1:b filterExpr: key is not null (type: boolean) @@ -134,27 +130,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) Reduce Operator Tree: Join Operator condition map: @@ -268,25 +260,21 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: int) - mode: final - outputColumnNames: _col0 + Group By Operator + keys: key (type: int) + mode: final + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: key Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: key + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: subq2:subq1:b filterExpr: key is not null (type: boolean) @@ -294,16 +282,12 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -436,27 +420,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) TableScan alias: src1:subq1:b filterExpr: key is not null (type: boolean) @@ -464,27 +444,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) Reduce Operator Tree: Join Operator condition map: @@ -511,27 +487,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) TableScan alias: src2:subq2:b filterExpr: key is not null (type: boolean) @@ -539,27 +511,23 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: int) - mode: final - outputColumnNames: _col0, _col1 + Group By Operator + aggregations: count() + keys: key (type: int) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint) + outputColumnNames: key, $f1 Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) - outputColumnNames: key, $f1 + Reduce Output Operator + key expressions: key (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: $f1 (type: bigint) + value expressions: $f1 (type: bigint) TableScan Reduce Output Operator key expressions: key (type: int) @@ -676,26 +644,21 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -778,26 +741,21 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -902,26 +860,21 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1016,26 +969,21 @@ STAGE PLANS: Filter Operator predicate: (key < 8) (type: boolean) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1312,26 +1260,21 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1406,28 +1349,23 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1518,26 +1456,21 @@ STAGE PLANS: Filter Operator predicate: (key < 6) (type: boolean) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - Select Operator - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - null sort order: - sort order: - value expressions: _col0 (type: bigint) + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + null sort order: + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1612,26 +1545,22 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: key, value, value0 - Select Operator - expressions: key (type: int), value (type: string), value0 (type: string) - outputColumnNames: key, val1, val2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: key, value, value0 + Select Operator + expressions: key (type: int), value (type: string), value0 (type: string) + outputColumnNames: key, val1, val2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1696,26 +1625,22 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Sorted Merge Bucket Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: key, value, value0 - Select Operator - expressions: key (type: int), value (type: string), value0 (type: string) - outputColumnNames: key, val1, val2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: key, value, value0 + Select Operator + expressions: key (type: int), value (type: string), value0 (type: string) + outputColumnNames: key, val1, val2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out index 2e037fdee9..9eed84d2e9 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out @@ -41,14 +41,10 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -60,46 +56,42 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: key, value, key0, value0 - Statistics: Num rows: 791 Data size: 281596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, key0, value0 + Statistics: Num rows: 791 Data size: 281596 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), UDFToInteger(key0) (type: int), value0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n112 Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), UDFToInteger(key0) (type: int), value0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string) + outputColumnNames: key1, value1, key2, value2 Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n112 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string) - outputColumnNames: key1, value1, key2, value2 - Statistics: Num rows: 791 Data size: 150290 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key1, 'hll'), compute_stats(value1, 'hll'), compute_stats(key2, 'hll'), compute_stats(value2, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + aggregations: compute_stats(key1, 'hll'), compute_stats(value1, 'hll'), compute_stats(key2, 'hll'), compute_stats(value2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/cbo_rp_cast1.q.out ql/src/test/results/clientpositive/cbo_rp_cast1.q.out new file mode 100644 index 0000000000..c4e76f59ec --- /dev/null +++ ql/src/test/results/clientpositive/cbo_rp_cast1.q.out @@ -0,0 +1,161 @@ +PREHOOK: query: CREATE TABLE dest1_n151(c1 INT, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 INT, c6 STRING, c7 INT) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n151 +POSTHOOK: query: CREATE TABLE dest1_n151(c1 INT, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 INT, c6 STRING, c7 INT) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n151 +PREHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n151 +POSTHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n151 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 5 (type: int), 5.0D (type: double), 5.0D (type: double), 5.0D (type: double), 5 (type: int), 'TRUE' (type: string), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 250 Data size: 31000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 31000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n151 + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: c1, c2, c3, c4, c5, c6, c7 + Statistics: Num rows: 250 Data size: 31000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 2984 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 2984 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5), compute_stats(VALUE._col6) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 3080 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3080 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n151 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, c2, c3, c4, c5, c6, c7 + Column Types: int, double, double, double, int, string, int + Table: default.dest1_n151 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n151 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n151 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n151 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n151 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT) + CAST(CAST(0 AS SMALLINT) AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n151 +POSTHOOK: Lineage: dest1_n151.c1 SIMPLE [] +POSTHOOK: Lineage: dest1_n151.c2 EXPRESSION [] +POSTHOOK: Lineage: dest1_n151.c3 EXPRESSION [] +POSTHOOK: Lineage: dest1_n151.c4 EXPRESSION [] +POSTHOOK: Lineage: dest1_n151.c5 SIMPLE [] +POSTHOOK: Lineage: dest1_n151.c6 EXPRESSION [] +POSTHOOK: Lineage: dest1_n151.c7 SIMPLE [] +PREHOOK: query: select dest1_n151.* FROM dest1_n151 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n151 +#### A masked pattern was here #### +POSTHOOK: query: select dest1_n151.* FROM dest1_n151 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n151 +#### A masked pattern was here #### +5 5.0 5.0 5.0 5 TRUE 1 diff --git ql/src/test/results/clientpositive/cbo_rp_cross_product_check_2.q.out ql/src/test/results/clientpositive/cbo_rp_cross_product_check_2.q.out index f445ba3d3d..9af443b26f 100644 --- ql/src/test/results/clientpositive/cbo_rp_cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/cbo_rp_cross_product_check_2.q.out @@ -159,14 +159,10 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -178,33 +174,29 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, key0, value0 + Statistics: Num rows: 20 Data size: 7040 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: key, value, key0, value0 - Statistics: Num rows: 20 Data size: 7040 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: key, value, key0, value0, key1, value1 + 0 + 1 + outputColumnNames: key, value, key0, value0, key1, value1 + Statistics: Num rows: 10000 Data size: 5300000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 10000 Data size: 5300000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10000 Data size: 5300000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Local Work: Map Reduce Local Work @@ -255,14 +247,10 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-3 Map Reduce @@ -274,34 +262,26 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: key - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: key - Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Local Work: Map Reduce Local Work @@ -423,22 +403,18 @@ STAGE PLANS: 1 outputColumnNames: key Statistics: Num rows: 100 Data size: 8600 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 100 Data size: 8600 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Local Work: Map Reduce Local Work @@ -694,14 +670,10 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce @@ -713,34 +685,26 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) outputColumnNames: key - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: key - Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1720 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/cbo_rp_join0.q.out ql/src/test/results/clientpositive/cbo_rp_join0.q.out index aca2f517d9..8d26e0bf28 100644 --- ql/src/test/results/clientpositive/cbo_rp_join0.q.out +++ ql/src/test/results/clientpositive/cbo_rp_join0.q.out @@ -33,17 +33,13 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + value expressions: c_int (type: int) TableScan alias: cbo_t2:cbo_t2 filterExpr: key is not null (type: boolean) @@ -51,17 +47,13 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + value expressions: c_int (type: int) TableScan alias: cbo_t3:cbo_t3 Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE @@ -697,17 +689,13 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + value expressions: c_int (type: int) TableScan alias: cbo_t2:cbo_t2 filterExpr: key is not null (type: boolean) @@ -715,17 +703,13 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + value expressions: c_int (type: int) TableScan alias: cbo_t3:cbo_t3 Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE @@ -746,17 +730,13 @@ STAGE PLANS: Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + value expressions: c_int (type: int) Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.out ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.out index 5bb0f03875..517d6056c3 100644 --- ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.out +++ ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.out @@ -43,19 +43,15 @@ STAGE PLANS: isSamplingPred: false predicate: ((UDFToDouble(key) < 20.0D) and (UDFToDouble(key) > 15.0D)) (type: boolean) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: value (type: string) - auto parallelism: false + tag: 0 + value expressions: value (type: string) + auto parallelism: false TableScan alias: b filterExpr: ((UDFToDouble(key) > 15.0D) and (UDFToDouble(key) < 20.0D)) (type: boolean) @@ -65,19 +61,15 @@ STAGE PLANS: isSamplingPred: false predicate: ((UDFToDouble(key) > 15.0D) and (UDFToDouble(key) < 20.0D)) (type: boolean) Statistics: Num rows: 111 Data size: 19758 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 111 Data size: 30192 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 111 Data size: 30192 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: value (type: string) - auto parallelism: false + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 111 Data size: 19758 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -357,19 +349,15 @@ STAGE PLANS: isSamplingPred: false predicate: ((UDFToDouble(key) < 20.0D) and (UDFToDouble(key) > 15.0D)) (type: boolean) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: value (type: string) - auto parallelism: false + tag: 0 + value expressions: value (type: string) + auto parallelism: false TableScan alias: b filterExpr: ((UDFToDouble(key) > 15.0D) and (UDFToDouble(key) < 20.0D)) (type: boolean) @@ -379,19 +367,15 @@ STAGE PLANS: isSamplingPred: false predicate: ((UDFToDouble(key) > 15.0D) and (UDFToDouble(key) < 20.0D)) (type: boolean) Statistics: Num rows: 111 Data size: 19758 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 111 Data size: 30192 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 111 Data size: 30192 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: value (type: string) - auto parallelism: false + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 111 Data size: 19758 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out index 0a7275f311..5f594258af 100644 --- ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out @@ -318,28 +318,32 @@ STAGE PLANS: mode: complete outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 + Select Operator + expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4 - columns.types struct:struct:struct:struct:struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4 + columns.types struct:struct:struct:struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false PREHOOK: query: FROM srcpart src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/llap/semijoin_hint.q.out ql/src/test/results/clientpositive/llap/semijoin_hint.q.out index 7303505b65..a578c5f04f 100644 --- ql/src/test/results/clientpositive/llap/semijoin_hint.q.out +++ ql/src/test/results/clientpositive/llap/semijoin_hint.q.out @@ -202,16 +202,12 @@ STAGE PLANS: Filter Operator predicate: (str is not null and str BETWEEN DynamicValue(RS_7_v_key1_min) AND DynamicValue(RS_7_v_key1_max) and in_bloom_filter(str, DynamicValue(RS_7_v_key1_bloom_filter))) (type: boolean) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str (type: string) - outputColumnNames: str + Reduce Output Operator + key expressions: str (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: str (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: str (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: str (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -223,31 +219,27 @@ STAGE PLANS: Filter Operator predicate: key1 is not null (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) + Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Select Operator expressions: key1 (type: string) - outputColumnNames: key1 + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=316) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=316) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 639 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 639 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 639 Basic stats: PARTIAL Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -260,19 +252,17 @@ STAGE PLANS: 0 str (type: string) 1 key1 (type: string) Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -348,16 +338,12 @@ STAGE PLANS: Filter Operator predicate: str is not null (type: boolean) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str (type: string) - outputColumnNames: str + Reduce Output Operator + key expressions: str (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: str (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: str (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: str (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -369,16 +355,12 @@ STAGE PLANS: Filter Operator predicate: key1 is not null (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: key1 + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -391,19 +373,17 @@ STAGE PLANS: 0 str (type: string) 1 key1 (type: string) Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -471,16 +451,12 @@ STAGE PLANS: Filter Operator predicate: cstring is not null (type: boolean) Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cstring (type: string) - outputColumnNames: cstring + Reduce Output Operator + key expressions: cstring (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: cstring (type: string) Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: cstring (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cstring (type: string) - Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map 6 @@ -492,17 +468,13 @@ STAGE PLANS: Filter Operator predicate: (str is not null and value is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str (type: string), value (type: string) - outputColumnNames: str, value + Reduce Output Operator + key expressions: value (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: value (type: string) Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: value (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: str (type: string) + value expressions: str (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map 7 @@ -514,16 +486,12 @@ STAGE PLANS: Filter Operator predicate: (key1 is not null and key1 BETWEEN DynamicValue(RS_9_srcpart_date_str_min) AND DynamicValue(RS_9_srcpart_date_str_max) and in_bloom_filter(key1, DynamicValue(RS_9_srcpart_date_str_bloom_filter))) (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: key1 + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -568,19 +536,17 @@ STAGE PLANS: 0 str (type: string) 1 key1 (type: string) Statistics: Num rows: 4521 Data size: 393327 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 4521 Data size: 393327 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: @@ -660,31 +626,27 @@ STAGE PLANS: Filter Operator predicate: cstring is not null (type: boolean) Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: cstring (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: cstring (type: string) + Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cstring (type: string) - outputColumnNames: cstring - Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: cstring (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: cstring (type: string) - Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 9174 Data size: 1287800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3000) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0 + Statistics: Num rows: 9174 Data size: 1287800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3000) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs Map 5 @@ -696,16 +658,12 @@ STAGE PLANS: Filter Operator predicate: (key1 is not null and key1 BETWEEN DynamicValue(RS_3_i_cstring_min) AND DynamicValue(RS_3_i_cstring_max) and in_bloom_filter(key1, DynamicValue(RS_3_i_cstring_bloom_filter))) (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: key1 + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Execution mode: vectorized, llap LLAP IO: all inputs Map 6 @@ -717,16 +675,12 @@ STAGE PLANS: Filter Operator predicate: str is not null (type: boolean) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str (type: string) - outputColumnNames: str + Reduce Output Operator + key expressions: str (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: str (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: str (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: str (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -741,19 +695,17 @@ STAGE PLANS: 1 key1 (type: string) 2 str (type: string) Statistics: Num rows: 20182 Data size: 1416580 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 20182 Data size: 1416580 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -831,31 +783,27 @@ STAGE PLANS: Filter Operator predicate: str is not null (type: boolean) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: str (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: str (type: string) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: str (type: string) - outputColumnNames: str - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: str (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: str (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs Map 5 @@ -867,16 +815,12 @@ STAGE PLANS: Filter Operator predicate: (key1 is not null and key1 BETWEEN DynamicValue(RS_3_k_str_min) AND DynamicValue(RS_3_k_str_max) and in_bloom_filter(key1, DynamicValue(RS_3_k_str_bloom_filter))) (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: key1 + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -889,19 +833,17 @@ STAGE PLANS: 0 str (type: string) 1 key1 (type: string) Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 2200 Data size: 191400 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -977,16 +919,12 @@ STAGE PLANS: Filter Operator predicate: value is not null (type: boolean) Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: value + Reduce Output Operator + key expressions: value (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: value (type: string) Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: value (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -998,16 +936,12 @@ STAGE PLANS: Filter Operator predicate: key1 is not null (type: boolean) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: key1 + Reduce Output Operator + key expressions: key1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key1 (type: string) Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: key1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key1 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: PARTIAL Column stats: PARTIAL Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -1020,19 +954,17 @@ STAGE PLANS: 0 value (type: string) 1 key1 (type: string) Statistics: Num rows: 2200 Data size: 200200 Basic stats: PARTIAL Column stats: NONE - Select Operator - Statistics: Num rows: 2200 Data size: 200200 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) + value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: